{ "updated": "2026-10-05T00:49:57.233828+00:00", "jev_version": "jev-1.13.0", "status": "live_run_and_priority_comparison_complete", "case_count": 17576, "api_progress": { "completed": 17576, "expected": 17576, "errors": 0, "elapsed_seconds_this_run": 2244.9661880000494, "last_suite": "norec_no", "workers": 4, "status": "complete", "billable_input_tokens": 10277935, "output_tokens": 1845946, "completed_attempts": 17577, "estimated_cost_usd": 0.43167327, "unconfirmed_or_inflight_reserved_usd": 0.126, "budget_accounted_usd": 0.55767327, "cap_usd": 25.0, "price_per_million_input_tokens_usd": 0.041999999999999996, "updated": "2026-10-01T08:41:43.892401+00:00", "completion_source": "Verified saved prediction records; historical worker progress retained separately." }, "spend": { "billable_input_tokens": 10917751, "output_tokens": 1934079, "completed_attempts": 18835, "estimated_cost_usd": 0.458545542, "unconfirmed_or_inflight_reserved_usd": 0.126, "budget_accounted_usd": 0.584545542, "cap_usd": 25.0, "price_per_million_input_tokens_usd": 0.041999999999999996, "updated": "2026-10-04T18:00:04.235462+00:00" }, "fixture_sha256": "7726ab88c63977be98ded59aa3cd9d4e37aa7cc1a870c0fcf04e240b00667f05", "models": { "jev-1.13.0": { "label": "Jev 1.13.0 ยท live API", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 178, "accuracy": 0.89, "wilson95": [ 0.839073090026985, 0.926227555398747 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 178, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.89, "brier": 0.17818099999999998, "nll": 1.2489526499582282, "ece10": 0.0587, "calibration_n": 200, "confident_wrong": 9, "macro_f1": 0.8925086945824273, "confusion_matrix": { "world": { "world": 44, "sports": 0, "business": 2, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 47, "sci_tech": 2, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 9, "sci_tech": 41, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 120, "accuracy": 0.6, "wilson95": [ 0.53083672039262, 0.6653942143319266 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 120, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.691005406591164, "nll": 7.027633429507393, "ece10": 0.2934974747474746, "calibration_n": 200, "confident_wrong": 48, "macro_f1": 0.5010205142372935, "confusion_matrix": { "sadness": { "sadness": 38, "joy": 2, "love": 3, "anger": 7, "fear": 7, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 14, "joy": 52, "love": 4, "anger": 5, "fear": 4, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 2, "joy": 6, "love": 4, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 8, "joy": 4, "love": 0, "anger": 14, "fear": 2, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 4, "joy": 3, "love": 0, "anger": 1, "fear": 11, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 196, "accuracy": 0.98, "wilson95": [ 0.9497129130941735, 0.9921955735836505 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 196, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.98, "brier": 0.04734799999999999, "nll": 0.1128579565479124, "ece10": 0.05820000000000003, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.9797550359348112, "confusion_matrix": { "false": { "false": 109, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 87, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": 0.7, "development": { "n": 92, "correct": 92, "wilson95": [ 0.9599186107143782, 1.0 ] }, "test_coverage": 0.955, "accepted_n": 191, "accepted_correct": 189, "accepted_accuracy": 0.9895287958115183, "accepted_wilson95": [ 0.9626309645142886, 0.997123705420151 ], "errors_per_1000_all_test_cases": 10.0, "note": "Selected on development only, applied unchanged to test. An empirical pilot estimate, not a production guarantee." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 179, "accuracy": 0.895, "wilson95": [ 0.8448196800887696, 0.9302925120731899 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 179, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.895, "brier": 0.15551500000000013, "nll": 0.2564608014270131, "ece10": 0.034350000000000006, "calibration_n": 200, "confident_wrong": 4, "macro_f1": 0.8817267888823181, "confusion_matrix": { "true": { "true": 56, "false": 21, "__failed__": 0 }, "false": { "true": 0, "false": 123, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.13745, "nll": 0.22297662406595173, "ece10": 0.065, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 3, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.26913333333333334, "nll": 0.4957784871304432, "ece10": 0.13166666666666668, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8533333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 1, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0013499999999999994, "nll": 0.007859223289270108, "ece10": 0.007499999999999997, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0002833333333333336, "nll": 0.004239360864479718, "ece10": 0.00416666666666667, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 91, "accuracy": 0.91, "wilson95": [ 0.837737871472837, 0.9519274599974348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 91, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.91, "brier": 0.14626799999999995, "nll": 0.564141080077806, "ece10": 0.07020000000000001, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.9046162752649198, "confusion_matrix": { "label_000": { "label_000": 25, "label_001": 0, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 7, "label_001": 16, "label_002": 1, "label_003": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.8393726744209776, "nll": 6.9641990412420895, "ece10": 0.3448888888888889, "calibration_n": 100, "confident_wrong": 17, "macro_f1": 0.4801587301587302, "confusion_matrix": { "label_000": { "label_000": 8, "label_001": 3, "label_002": 0, "label_003": 1, "label_004": 5, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 11, "label_002": 0, "label_003": 0, "label_004": 4, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 11, "label_003": 0, "label_004": 4, "label_005": 1, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 1, "label_002": 3, "label_003": 6, "label_004": 7, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 2, "label_002": 5, "label_003": 0, "label_004": 9, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 0, "label_002": 6, "label_003": 0, "label_004": 6, "label_005": 3, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 89, "accuracy": 0.89, "wilson95": [ 0.8136870349691969, 0.9374580364293543 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 89, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.89, "brier": 0.17575328905213752, "nll": 1.2673767677863577, "ece10": 0.046485858585858596, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.8833333333333333, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 2, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 1, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 2, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 2, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00020416666666666692, "nll": 0.004427045314567113, "ece10": 0.004375000000000004, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 81, "accuracy": 0.7297297297297297, "wilson95": [ 0.6404595664697532, 0.8036309259684178 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 81, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7297297297297297, "brier": 0.36575315315315304, "nll": 0.6216555764610855, "ece10": 0.09603603603603603, "calibration_n": 111, "confident_wrong": 2 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 70, "accuracy": 0.9722222222222222, "wilson95": [ 0.9042582477856177, 0.9923489780955054 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 70, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9722222222222222, "cluster_bootstrap95": [ 0.9305555555555556, 1.0 ], "independent_source_groups": 36, "brier": 0.02963611111111111, "nll": 0.06008214523798979, "ece10": 0.04791666666666667, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.11445263157894736, "nll": 0.21292358902107586, "ece10": 0.11000000000000001, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9467787114845938, "confusion_matrix": { "false": { "false": 10, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0297375, "nll": 0.0936086134071683, "ece10": 0.08437500000000002, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.1769454545454545, "nll": 0.27129857895038106, "ece10": 0.14909090909090908, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.905982905982906, "confusion_matrix": { "false": { "false": 8, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0, "nll": 0.0, "ece10": 0.0, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0034666666666666674, "nll": 0.02318250452267727, "ece10": 0.02222222222222223, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0005263157894736839, "nll": 0.00434847519412299, "ece10": 0.004210526315789472, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.09670588235294117, "nll": 0.14538236334686783, "ece10": 0.09176470588235294, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9206349206349206, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 3, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0386, "nll": 0.07693169584600905, "ece10": 0.06333333333333334, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 2, "test": 0, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 2, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03523529411764706, "nll": 0.08386635762702253, "ece10": 0.07117647058823531, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.07828750000000001, "nll": 0.13968528401945932, "ece10": 0.108125, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9372549019607843, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.014677777777777774, "nll": 0.05153154458990642, "ece10": 0.047222222222222214, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "harassment": { "harassment": 5, "threat": 0, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 2, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 6, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 21, "accuracy": 0.9545454545454546, "wilson95": [ 0.7820195416134267, 0.991930572378736 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9545454545454546, "brier": 0.044472727272727275, "nll": 0.06428041696822494, "ece10": 0.04272727272727274, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 1, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 3, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 1.1764705882352954e-05, "nll": 0.0005911962266765559, "ece10": 0.0005882352941176475, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0, "nll": 0.0, "ece10": 0.0, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0274, "nll": 0.09905921067951251, "ece10": 0.09117647058823532, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0, "nll": 0.0, "ece10": 0.0, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.013466666666666667, "nll": 0.03211682508535225, "ece10": 0.028, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "meeting_request": { "meeting_request": 3, "question": 0, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.007555555555555556, "nll": 0.022182458279615735, "ece10": 0.019999999999999997, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 3, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.06436666666666666, "nll": 0.15517193713619604, "ece10": 0.1327777777777778, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9442724458204335, "confusion_matrix": { "false": { "false": 8, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.0798375, "nll": 0.14569117560101627, "ece10": 0.11687500000000002, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9415584415584416, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.06847777777777778, "nll": 0.13675574332524987, "ece10": 0.11277777777777778, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9407407407407407, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 18, "accuracy": 0.9, "wilson95": [ 0.6989663547715128, 0.9721335187862319 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.15800999999999998, "nll": 0.30200585603717134, "ece10": 0.093, "calibration_n": 20, "confident_wrong": 1, "macro_f1": 0.891024531024531, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 4, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 1, "harassment": 0, "boosting": 0, "smurfing": 2, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 18, "accuracy": 0.8571428571428571, "wilson95": [ 0.6536393971937906, 0.9501898750027473 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.17162857142857144, "nll": 0.29530828238781426, "ece10": 0.15809523809523807, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.8592063492063492, "confusion_matrix": { "spelling": { "spelling": 4, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 3, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 4, "punctuation": 0, "none": 0, "__failed__": 0 }, "punctuation": { "spelling": 1, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 3, "none": 1, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.039612499999999995, "nll": 0.12656736894202766, "ece10": 0.114375, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 7.826086956521747e-05, "nll": 0.0013243133689003728, "ece10": 0.0013043478260869577, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 14, "accuracy": 1.0, "wilson95": [ 0.7846891972623643, 1.0 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.015357142857142856, "nll": 0.046646805290985874, "ece10": 0.04214285714285714, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.13872500000000001, "nll": 0.23239086706532017, "ece10": 0.14375, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8740079365079365, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.1089875, "nll": 0.17934506236664075, "ece10": 0.09187500000000001, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0021647058823529436, "nll": 0.027619549513093103, "ece10": 0.027058823529411788, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00022352941176470608, "nll": 0.004174882475338795, "ece10": 0.004117647058823533, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0028823529411764713, "nll": 0.013248571618737319, "ece10": 0.01235294117647059, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.015199999999999998, "nll": 0.07476073242001945, "ece10": 0.07058823529411766, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.017412499999999994, "nll": 0.06696667092887311, "ece10": 0.06187500000000001, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.031200000000000002, "nll": 0.0797592931169954, "ece10": 0.07, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0005000000000000004, "nll": 0.005129700622377561, "ece10": 0.0050000000000000044, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 4, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.19876470588235293, "nll": 0.3636620014381149, "ece10": 0.2970588235294117, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0016888888888888882, "nll": 0.013778152547269678, "ece10": 0.013333333333333332, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 4, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.12301111111111106, "nll": 0.23182859661346938, "ece10": 0.10333333333333332, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.10828750000000002, "nll": 0.18621685122702508, "ece10": 0.14062500000000003, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9352226720647774, "confusion_matrix": { "false": { "false": 9, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03729333333333333, "nll": 0.09662479872376022, "ece10": 0.08466666666666668, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.016058823529411764, "nll": 0.04194162053776807, "ece10": 0.03705882352941177, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 4, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0471764705882353, "nll": 0.1458432037665872, "ece10": 0.13176470588235292, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 17, "accuracy": 0.8947368421052632, "wilson95": [ 0.6860591728889099, 0.9706414397011299 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8947368421052632, "brier": 0.1381157894736842, "nll": 0.24194940053800437, "ece10": 0.10368421052631578, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8802641802641803, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0721529411764706, "nll": 0.19841125452962297, "ece10": 0.17705882352941174, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.012155555555555558, "nll": 0.05237367126957894, "ece10": 0.048888888888888905, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.09918999999999999, "nll": 0.1776825186109318, "ece10": 0.09200000000000001, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9476190476190476, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 20, "accuracy": 1.0, "wilson95": [ 0.8388748419471806, 1.0 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.01157, "nll": 0.021278288990758364, "ece10": 0.017499999999999998, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 16, "accuracy": 0.8, "wilson95": [ 0.5839825677481064, 0.919342337420202 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.25612, "nll": 0.4226986863155062, "ece10": 0.177, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.797979797979798, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.14281818181818184, "nll": 0.22653281470269668, "ece10": 0.09772727272727275, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9113247863247864, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 629, "accuracy": 0.6988888888888889, "wilson95": [ 0.6681248743327715, 0.7279622895234958 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 791, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7371854613233924, "cluster_bootstrap95": [ 0.6619267659670504, 0.7405337862870147 ], "independent_source_groups": 596, "brier": 0.3652678726983856, "nll": 1.7155543617600304, "ece10": 0.0883492897286001, "calibration_n": 1073, "confident_wrong": 44 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 465, "accuracy": 0.8101045296167247, "wilson95": [ 0.7759970981093772, 0.8400888447832525 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 820, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8760683760683761, "brier": 0.17648694510066607, "nll": 0.5003724127661614, "ece10": 0.03767374600707929, "calibration_n": 936, "confident_wrong": 30 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 490, "accuracy": 0.7, "wilson95": [ 0.6650360327155247, 0.7327808285832509 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 849, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7803308823529411, "cluster_bootstrap95": [ 0.66619302853172, 0.7333380084151472 ], "independent_source_groups": 650, "brier": 0.3044224317596652, "nll": 0.6779350436967428, "ece10": 0.027878880718954245, "calibration_n": 1088, "confident_wrong": 26 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 97, "accuracy": 0.2425, "wilson95": [ 0.2030764095091307, 0.286822422073032 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1473, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7365, "brier": 0.36503387898173684, "nll": 0.7404277484022564, "ece10": 0.03703111111111111, "calibration_n": 2000, "confident_wrong": 42 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 5.0000000000000057e-05, "nll": 0.0017983469599294221, "ece10": 0.0017857142857142874, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 1.42857142857143e-05, "nll": 0.0007178811323929607, "ece10": 0.0007142857142857149, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0006500000000000001, "nll": 0.007318374650411501, "ece10": 0.007142857142857145, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00011428571428571438, "nll": 0.0036003380488058645, "ece10": 0.003571428571428575, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.015215000000000003, "nll": 0.03390557131220365, "ece10": 0.027000000000000003, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "review": { "review": 8, "it": 0, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.014335, "nll": 0.030162837617740856, "ece10": 0.022250000000000002, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 8, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 39, "accuracy": 0.975, "wilson95": [ 0.8711863103652591, 0.9955731684973186 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.975, "brier": 0.031815, "nll": 0.06976913434223021, "ece10": 0.037000000000000005, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9749019607843138, "confusion_matrix": { "review": { "review": 7, "it": 0, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 39, "accuracy": 0.975, "wilson95": [ 0.8711863103652591, 0.9955731684973186 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.975, "brier": 0.03209, "nll": 0.059967056660596964, "ece10": 0.025750000000000002, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9749019607843138, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 237, "accuracy": 0.5925, "wilson95": [ 0.5436883047891221, 0.639551920759033 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 237, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5925, "brier": 0.6567524514845426, "nll": 5.838163042553691, "ece10": 0.2772358585858584, "calibration_n": 400, "confident_wrong": 67, "macro_f1": 0.49296716360434506, "confusion_matrix": { "sadness": { "sadness": 87, "joy": 11, "love": 6, "anger": 5, "fear": 10, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 21, "joy": 75, "love": 11, "anger": 6, "fear": 3, "surprise": 2, "__failed__": 0 }, "love": { "sadness": 9, "joy": 12, "love": 10, "anger": 3, "fear": 2, "surprise": 1, "__failed__": 0 }, "anger": { "sadness": 14, "joy": 7, "love": 4, "anger": 34, "fear": 2, "surprise": 3, "__failed__": 0 }, "fear": { "sadness": 12, "joy": 5, "love": 0, "anger": 3, "fear": 29, "surprise": 3, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 2, "love": 1, "anger": 1, "fear": 2, "surprise": 2, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 277, "accuracy": 0.9233333333333333, "wilson95": [ 0.8875898961498251, 0.9483723882133924 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 277, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9233333333333333, "brier": 0.10723483896201062, "nll": 0.8398065939141014, "ece10": 0.021789562289562296, "calibration_n": 300, "confident_wrong": 3 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 377, "accuracy": 0.9425, "wilson95": [ 0.9151991622718807, 0.9613824572422669 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 377, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9425, "brier": 0.10462649999999998, "nll": 0.18863968869968215, "ece10": 0.04632499999999999, "calibration_n": 400, "confident_wrong": 3, "macro_f1": 0.9214688735051344, "confusion_matrix": { "false": { "false": 292, "true": 17, "__failed__": 0 }, "true": { "false": 6, "true": 85, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 346, "accuracy": 0.865, "wilson95": [ 0.8280190157519688, 0.8950370093839947 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 346, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.865, "brier": 0.23149622074278128, "nll": 2.20207180566066, "ece10": 0.09364545454545455, "calibration_n": 400, "confident_wrong": 33, "macro_f1": 0.8541068443051201, "confusion_matrix": { "world": { "world": 86, "sports": 6, "business": 10, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 2, "sports": 121, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 68, "sci_tech": 2, "__failed__": 0 }, "sci_tech": { "world": 7, "sports": 0, "business": 24, "sci_tech": 71, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 360, "accuracy": 0.9, "wilson95": [ 0.8666894236725972, 0.9257007408599657 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 360, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.15292700000000004, "nll": 0.2506925422736928, "ece10": 0.042550000000000004, "calibration_n": 400, "confident_wrong": 2, "macro_f1": 0.8877791493659521, "confusion_matrix": { "true": { "true": 114, "false": 39, "__failed__": 0 }, "false": { "true": 1, "false": 246, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 225, "accuracy": 0.75, "wilson95": [ 0.6980484035459826, 0.7956301108417443 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 225, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.363040532462674, "nll": 2.411807869718936, "ece10": 0.1434855218855219, "calibration_n": 300, "confident_wrong": 20 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 393, "accuracy": 0.9825, "wilson95": [ 0.964322927074442, 0.991497708892962 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 393, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9825, "brier": 0.0373979999999999, "nll": 0.09636286934307937, "ece10": 0.05804999999999999, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.9824516323115586, "confusion_matrix": { "false": { "false": 186, "true": 2, "__failed__": 0 }, "true": { "false": 5, "true": 207, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 265, "accuracy": 0.6625, "wilson95": [ 0.6148101420626579, 0.7070983622786957 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 265, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6625, "brier": 0.4311780000000003, "nll": 0.6188653582447352, "ece10": 0.17585, "calibration_n": 400, "confident_wrong": 23, "macro_f1": 0.6285059748622377, "confusion_matrix": { "false": { "false": 193, "true": 7, "__failed__": 0 }, "true": { "false": 128, "true": 72, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 75, "accuracy": 0.75, "wilson95": [ 0.656955364519384, 0.8245478863771232 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 75, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.38416799999999995, "nll": 1.897652082878465, "ece10": 0.14400000000000004, "calibration_n": 100, "confident_wrong": 10, "macro_f1": 0.7270444371656295, "confusion_matrix": { "yes": { "yes": 52, "no": 2, "__failed__": 0 }, "no": { "yes": 23, "no": 23, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 56, "accuracy": 0.56, "wilson95": [ 0.4622810465167699, 0.6532797336983921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 56, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.56, "brier": 0.7763863475155596, "nll": 8.343700874496754, "ece10": 0.307050505050505, "calibration_n": 100, "confident_wrong": 22, "macro_f1": 0.5712153124653125, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 2, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 2, "dispute_charge": 2, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 3, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 8, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 2, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 3, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 5, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 5, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 4, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 7, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 4, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 4, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 3, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 146, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.73, "brier": 0.43191380716253436, "nll": 1.0447845902982729, "ece10": 0.1972626262626263, "calibration_n": 200, "confident_wrong": 32 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 133, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.665, "brier": 0.5791817521681463, "nll": 5.241764306386196, "ece10": 0.252040404040404, "calibration_n": 200, "confident_wrong": 40 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 79, "accuracy": 0.79, "wilson95": [ 0.7002003116591013, 0.858343459380847 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 79, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.3427799999999999, "nll": 3.752704665714816, "ece10": 0.1542, "calibration_n": 100, "confident_wrong": 11, "macro_f1": 0.7621378893437717, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 13, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 2, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 12, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 7, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 6, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 4, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 6, "accuracy": 0.06, "wilson95": [ 0.027786123963188192, 0.1247681544589592 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 213, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5325, "brier": 0.7674337591572301, "nll": 6.576880839638409, "ece10": 0.3302484848484846, "calibration_n": 400, "confident_wrong": 93 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 64, "accuracy": 0.64, "wilson95": [ 0.5423540245160874, 0.7272877959859567 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 64, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.64, "brier": 0.5365588829711256, "nll": 4.776184483586033, "ece10": 0.1794818181818182, "calibration_n": 100, "confident_wrong": 7, "macro_f1": 0.6448293716946968, "confusion_matrix": { "politics": { "politics": 6, "business": 0, "technology": 0, "science": 1, "health": 3, "sports": 0, "entertainment": 0, "world": 2, "culture": 1, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 1, "business": 1, "technology": 7, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 2, "business": 1, "technology": 0, "science": 4, "health": 2, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 1, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 5, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 1, "weather": 1, "__failed__": 0 }, "world": { "politics": 2, "business": 1, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 4, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 1, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 7, "weather": 1, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 7, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 37, "accuracy": 0.37, "wilson95": [ 0.28182360534324524, 0.4677947041905709 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.37, "brier": 0.9556326183042546, "nll": 9.126871058030966, "ece10": 0.398449494949495, "calibration_n": 100, "confident_wrong": 18, "macro_f1": 0.33063233860110963, "confusion_matrix": { "biology": { "biology": 6, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 9, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 1, "medicine": 1, "chemistry": 2, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 5, "medicine": 2, "chemistry": 0, "physics": 4, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 2, "economics": 1, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 1, "medicine": 3, "chemistry": 1, "physics": 0, "computer_science": 2, "mathematics": 0, "earth_science": 2, "engineering": 6, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 1, "computer_science": 1, "mathematics": 0, "earth_science": 2, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 2, "medicine": 4, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 5, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 5, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 4, "other": 0, "__failed__": 0 }, "other": { "biology": 1, "medicine": 0, "chemistry": 0, "physics": 2, "computer_science": 1, "mathematics": 1, "earth_science": 3, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 16, "accuracy": 0.16, "wilson95": [ 0.10095288488804782, 0.24420269389270244 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 76, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.38, "brier": 0.7382439999999998, "nll": 8.406825046509626, "ece10": 0.3562999999999999, "calibration_n": 100, "confident_wrong": 26 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 34, "accuracy": 0.34, "wilson95": [ 0.25461520797348164, 0.43722271145275377 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 124, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.62, "brier": 0.2770228829711254, "nll": 1.734263919788606, "ece10": 0.14151818181818182, "calibration_n": 100, "confident_wrong": 8 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 83, "accuracy": 0.83, "wilson95": [ 0.7445199523239887, 0.8910643388594006 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 83, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.83, "brier": 0.26627799999999996, "nll": 1.7321500932926583, "ece10": 0.12390000000000002, "calibration_n": 100, "confident_wrong": 9, "macro_f1": 0.8316626570435712, "confusion_matrix": { "negative": { "negative": 35, "neutral": 1, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 12, "neutral": 21, "positive": 0, "__failed__": 0 }, "positive": { "negative": 3, "neutral": 1, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 21, "accuracy": 0.21, "wilson95": [ 0.141656540619153, 0.29979968834089865 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 104, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.25175000000000003, "nll": 0.6846593491157309, "ece10": 0.1041, "calibration_n": 100, "confident_wrong": 4 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 51, "accuracy": 0.51, "wilson95": [ 0.4134801193402717, 0.6057800106955886 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 236, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7866666666666666, "brier": 0.3464901740638712, "nll": 2.190736167611181, "ece10": 0.13571245791245787, "calibration_n": 300, "confident_wrong": 35 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 65, "accuracy": 0.65, "wilson95": [ 0.5525444329688796, 0.7363575175690247 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 65, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.5377048231813081, "nll": 5.436186194508561, "ece10": 0.18655050505050502, "calibration_n": 100, "confident_wrong": 9, "macro_f1": 0.6332269271344904, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 2, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 4, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 2, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 4, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 3, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 3, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 5, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 5, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 1, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 1, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 39, "accuracy": 0.39, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.39, "brier": 0.9709989752066113, "nll": 10.501385438791845, "ece10": 0.44849393939393933, "calibration_n": 100, "confident_wrong": 26, "macro_f1": 0.38556805325437143, "confusion_matrix": { "billing": { "billing": 5, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 6, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 3, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 3, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 4, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 3, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 2, "integrations": 4, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 6, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "compliance": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 3, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 3, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 4, "other": 0, "__failed__": 0 }, "other": { "billing": 4, "account_access": 0, "shipping": 1, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 9, "accuracy": 0.09, "wilson95": [ 0.04807254000256516, 0.1622621285271631 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 145, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48333333333333334, "brier": 0.8014453904703607, "nll": 6.9175348395293215, "ece10": 0.3391653198653198, "calibration_n": 300, "confident_wrong": 49 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 59, "accuracy": 0.59, "wilson95": [ 0.4920143005341078, 0.6813268697886348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 59, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.59, "brier": 0.6937439999999997, "nll": 7.8923505887528265, "ece10": 0.2846, "calibration_n": 100, "confident_wrong": 20, "macro_f1": 0.5509259746101852, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 4, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 1, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 2, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 7, "baggage": 0, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 11, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 3, "loyalty": 2, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 1, "loyalty": 6, "disruption": 1, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 6, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 3, "car": 2, "visa_docs": 2, "loyalty": 1, "disruption": 2, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 91, "accuracy": 0.91, "wilson95": [ 0.837737871472837, 0.9519274599974348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 91, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.91, "brier": 0.14586800000000003, "nll": 0.5838512199502697, "ece10": 0.059200000000000016, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.9046162752649198, "confusion_matrix": { "business": { "business": 25, "science and technology": 0, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 7, "science and technology": 16, "sports": 1, "world": 1, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.17959061218243036, "nll": 1.276051172023854, "ece10": 0.0702070707070707, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.8699074074074074, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 2, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 1, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 2, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 2, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 1, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.8400070584634223, "nll": 7.851844711164622, "ece10": 0.34728181818181814, "calibration_n": 100, "confident_wrong": 20, "macro_f1": 0.46357055956835563, "confusion_matrix": { "anger": { "anger": 9, "fear": 3, "joy": 0, "love": 1, "sadness": 4, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 1, "fear": 12, "joy": 0, "love": 0, "sadness": 3, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 11, "love": 0, "sadness": 4, "surprise": 1, "__failed__": 0 }, "love": { "anger": 1, "fear": 1, "joy": 3, "love": 5, "sadness": 7, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 1, "fear": 3, "joy": 3, "love": 0, "sadness": 9, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 1, "joy": 6, "love": 0, "sadness": 6, "surprise": 2, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2550, "accuracy": 0.8649932157394844, "wilson95": [ 0.8521812704161608, 0.8768551734515789 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2550, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8649932157394844, "brier": 0.203658968050362, "nll": 1.2727918620890906, "ece10": 0.048638282417115444, "calibration_n": 2948, "confident_wrong": 100, "macro_f1": 0.8611463250413053, "confusion_matrix": { "alarm": { "alarm": 94, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 0, "audio": 52, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 3, "audio": 0, "calendar": 358, "cooking": 0, "datetime": 11, "email": 1, "general": 4, "iot": 0, "lists": 5, "music": 1, "news": 3, "play": 1, "qa": 6, "recommendation": 4, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 65, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 98, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 0, "email": 231, "general": 24, "iot": 0, "lists": 7, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 6, "datetime": 0, "email": 0, "general": 105, "iot": 3, "lists": 1, "music": 1, "news": 7, "play": 3, "qa": 29, "recommendation": 16, "social": 3, "takeaway": 0, "transport": 3, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 11, "datetime": 0, "email": 0, "general": 7, "iot": 186, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 6, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 8, "iot": 2, "lists": 125, "music": 4, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 0, "lists": 1, "music": 68, "news": 0, "play": 4, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 113, "play": 1, "qa": 7, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 12, "iot": 3, "lists": 0, "music": 31, "news": 1, "play": 332, "qa": 1, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 16, "iot": 0, "lists": 0, "music": 1, "news": 10, "play": 0, "qa": 249, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 4, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 5, "play": 1, "qa": 3, "recommendation": 80, "social": 0, "takeaway": 0, "transport": 2, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 4, "general": 13, "iot": 0, "lists": 0, "music": 0, "news": 4, "play": 1, "qa": 0, "recommendation": 0, "social": 83, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 2, "social": 0, "takeaway": 50, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 119, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 2, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 142, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2412, "accuracy": 0.8181818181818182, "wilson95": [ 0.8038478031942561, 0.8316876841179558 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2412, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8181818181818182, "brier": 0.266521286228234, "nll": 1.4200927221055077, "ece10": 0.05007740224497344, "calibration_n": 2948, "confident_wrong": 97, "macro_f1": 0.8158182297894032, "confusion_matrix": { "alarm": { "alarm": 94, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 2, "audio": 46, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 9, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 5, "audio": 1, "calendar": 342, "cooking": 0, "datetime": 13, "email": 1, "general": 9, "iot": 0, "lists": 3, "music": 2, "news": 7, "play": 0, "qa": 4, "recommendation": 7, "social": 2, "takeaway": 1, "transport": 2, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 66, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 99, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 1, "audio": 0, "calendar": 4, "cooking": 1, "datetime": 0, "email": 223, "general": 21, "iot": 0, "lists": 9, "music": 0, "news": 3, "play": 0, "qa": 3, "recommendation": 0, "social": 3, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 7, "cooking": 6, "datetime": 1, "email": 1, "general": 109, "iot": 2, "lists": 1, "music": 1, "news": 9, "play": 3, "qa": 26, "recommendation": 11, "social": 0, "takeaway": 0, "transport": 5, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 3, "audio": 4, "calendar": 1, "cooking": 12, "datetime": 0, "email": 0, "general": 10, "iot": 178, "lists": 1, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 4, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 2, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 0, "lists": 124, "music": 3, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 1, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 1, "audio": 2, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 1, "music": 66, "news": 0, "play": 4, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 108, "play": 0, "qa": 9, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "play": { "alarm": 0, "audio": 7, "calendar": 1, "cooking": 1, "datetime": 4, "email": 0, "general": 21, "iot": 5, "lists": 2, "music": 82, "news": 5, "play": 250, "qa": 4, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 1, "datetime": 0, "email": 0, "general": 19, "iot": 1, "lists": 0, "music": 4, "news": 6, "play": 0, "qa": 248, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 4, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 2, "play": 1, "qa": 3, "recommendation": 82, "social": 0, "takeaway": 0, "transport": 2, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 1, "calendar": 2, "cooking": 0, "datetime": 0, "email": 3, "general": 13, "iot": 2, "lists": 0, "music": 0, "news": 3, "play": 1, "qa": 1, "recommendation": 0, "social": 79, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 2, "datetime": 1, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 49, "transport": 1, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 116, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 7, "iot": 2, "lists": 4, "music": 0, "news": 1, "play": 0, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 133, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 843, "accuracy": 0.7186700767263428, "wilson95": [ 0.692256510191496, 0.7436560727269154 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 843, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7186700767263428, "cluster_bootstrap95": [ 0.6864316202301844, 0.7498219696134694 ], "independent_source_groups": 47, "brier": 0.41624878180654445, "nll": 1.4456929337177027, "ece10": 0.11748775048007784, "calibration_n": 1173, "confident_wrong": 85, "macro_f1": 0.700077795574208, "confusion_matrix": { "negative": { "negative": 142, "neutral": 23, "positive": 17, "__failed__": 0 }, "neutral": { "negative": 118, "neutral": 374, "positive": 98, "__failed__": 0 }, "positive": { "negative": 35, "neutral": 39, "positive": 327, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "clm-int8": { "label": "CLM 8B INT8", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 93, "accuracy": 0.465, "wilson95": [ 0.3971856415700402, 0.5341335312763685 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 93, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.465, "brier": 0.7820199925434922, "nll": 1.98391807772799, "ece10": 0.2666480150451842, "calibration_n": 200, "confident_wrong": 18, "macro_f1": 0.40342828374691436, "confusion_matrix": { "world": { "world": 16, "sports": 0, "business": 1, "sci_tech": 33, "__failed__": 0 }, "sports": { "world": 8, "sports": 0, "business": 2, "sci_tech": 37, "__failed__": 0 }, "business": { "world": 5, "sports": 0, "business": 33, "sci_tech": 13, "__failed__": 0 }, "sci_tech": { "world": 7, "sports": 0, "business": 1, "sci_tech": 44, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 51, "accuracy": 0.255, "wilson95": [ 0.1996049517203135, 0.3196292582045472 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 51, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.255, "brier": 1.1277203912286777, "nll": 5.489316586222015, "ece10": 0.3599839863477638, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.10025220680958387, "confusion_matrix": { "sadness": { "sadness": 48, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 9, "__failed__": 0 }, "joy": { "sadness": 40, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 39, "__failed__": 0 }, "love": { "sadness": 7, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 5, "__failed__": 0 }, "anger": { "sadness": 15, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 13, "__failed__": 0 }, "fear": { "sadness": 15, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 5, "__failed__": 0 }, "surprise": { "sadness": 1, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 3, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 132, "accuracy": 0.66, "wilson95": [ 0.5918849552683626, 0.7220845402909121 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 132, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.66, "brier": 0.4520064525809723, "nll": 0.6439515275071809, "ece10": 0.09162244534429652, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.6594551282051282, "confusion_matrix": { "false": { "false": 70, "true": 39, "__failed__": 0 }, "true": { "false": 29, "true": 62, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 80, "accuracy": 0.4, "wilson95": [ 0.33460578566807336, 0.46916327960737997 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.7767329446247961, "nll": 1.073758233363038, "ece10": 0.42974870765270223, "calibration_n": 200, "confident_wrong": 21, "macro_f1": 0.3162393162393162, "confusion_matrix": { "true": { "true": 75, "false": 2, "__failed__": 0 }, "false": { "true": 118, "false": 5, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 2, "accuracy": 0.16666666666666666, "wilson95": [ 0.04696514218385378, 0.4480308622529734 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 2, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.16666666666666666, "brier": 1.5763529980119302, "nll": 4.77543590979621, "ece10": 0.7915249660109435, "calibration_n": 12, "confident_wrong": 8, "macro_f1": 0.05714285714285714, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "review": { "maintenance": 4, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 2, "accuracy": 0.16666666666666666, "wilson95": [ 0.04696514218385378, 0.4480308622529734 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 2, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.16666666666666666, "brier": 1.0049918795229702, "nll": 2.129447704909183, "ece10": 0.3725159852801716, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.16969696969696968, "confusion_matrix": { "maintenance": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 1, "procurement": 1, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "review": { "maintenance": 4, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 6, "accuracy": 0.5, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.8190691704624963, "nll": 1.839050893375064, "ece10": 0.4048683840074743, "calibration_n": 12, "confident_wrong": 3, "macro_f1": 0.35064935064935066, "confusion_matrix": { "design": { "design": 0, "inspection": 0, "purchase": 1, "work": 2, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 4, "accuracy": 0.3333333333333333, "wilson95": [ 0.13812009109121315, 0.6093779111272004 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3333333333333333, "brier": 0.7976119770805391, "nll": 1.4718038123498023, "ece10": 0.21810200481685152, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.23214285714285715, "confusion_matrix": { "design": { "design": 1, "inspection": 0, "purchase": 0, "work": 2, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 47, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47, "brier": 0.6306514566896154, "nll": 1.131925830138534, "ece10": 0.18336878481634103, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.42125795479919614, "confusion_matrix": { "label_000": { "label_000": 13, "label_001": 0, "label_002": 11, "label_003": 1, "__failed__": 0 }, "label_001": { "label_000": 8, "label_001": 4, "label_002": 12, "label_003": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 1, "label_001": 0, "label_002": 19, "label_003": 5, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 22, "accuracy": 0.22, "wilson95": [ 0.150012823846054, 0.31070353514985793 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.22, "brier": 0.9806249998594362, "nll": 2.2123536026705954, "ece10": 0.34438478548598284, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.12657798643714135, "confusion_matrix": { "label_000": { "label_000": 0, "label_001": 0, "label_002": 7, "label_003": 0, "label_004": 10, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 1, "label_002": 1, "label_003": 0, "label_004": 15, "label_005": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 11, "label_003": 0, "label_004": 6, "label_005": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 11, "label_003": 0, "label_004": 6, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 6, "label_003": 0, "label_004": 10, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 7, "label_003": 1, "label_004": 8, "label_005": 0, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.08, "brier": 0.9869446384183358, "nll": 4.249826097414274, "ece10": 0.11632673403685337, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.04681187150699346, "confusion_matrix": { "label_000": { "label_000": 0, "label_001": 0, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 1, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 2, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 1, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 33, "accuracy": 0.6875, "wilson95": [ 0.5466692858410841, 0.8005431655341219 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 33, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.46896840721589483, "nll": 0.8042333025448544, "ece10": 0.2084968025081885, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 38, "accuracy": 0.34234234234234234, "wilson95": [ 0.26067404972026387, 0.43455796527314816 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.34234234234234234, "brier": 0.9896565841052514, "nll": 2.314103681863784, "ece10": 0.4274882770065715, "calibration_n": 111, "confident_wrong": 22 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 25, "accuracy": 0.3472222222222222, "wilson95": [ 0.24753423008275116, 0.46238696154394426 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 25, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3472222222222222, "cluster_bootstrap95": [ 0.20833333333333334, 0.4861111111111111 ], "independent_source_groups": 36, "brier": 0.9331354556870747, "nll": 2.2745269011055145, "ece10": 0.42648159950188874, "calibration_n": 72, "confident_wrong": 11 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 9, "accuracy": 0.47368421052631576, "wilson95": [ 0.2732980516117956, 0.6829219075488682 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47368421052631576, "brier": 0.5132845520678682, "nll": 0.7001622785128533, "ece10": 0.27683453873097014, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.391025641025641, "confusion_matrix": { "false": { "false": 1, "true": 10, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.5610401072797905, "nll": 0.7603419085259099, "ece10": 0.15679762437062994, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.3333333333333333, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 8, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 10, "accuracy": 0.45454545454545453, "wilson95": [ 0.2692029418081331, 0.6534020467926505 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45454545454545453, "brier": 0.5897541535479064, "nll": 0.7861646821954409, "ece10": 0.30779466679541045, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.37142857142857144, "confusion_matrix": { "false": { "false": 1, "true": 9, "__failed__": 0 }, "true": { "false": 3, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 3, "accuracy": 0.1875, "wilson95": [ 0.06591599071428142, 0.4300888096197414 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 3, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.1875, "brier": 1.3368244752921452, "nll": 2.7641381787879777, "ece10": 0.6738255122445598, "calibration_n": 16, "confident_wrong": 6, "macro_f1": 0.08333333333333333, "confusion_matrix": { "bug_report": { "bug_report": 0, "feature_request": 1, "praise": 4, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 0, "praise": 4, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 4, "pricing_billing": 0, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 3, "accuracy": 0.16666666666666666, "wilson95": [ 0.05836576805861446, 0.39222038103915724 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 3, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.16666666666666666, "brier": 1.636084253140603, "nll": 4.922858495869762, "ece10": 0.819610361923048, "calibration_n": 18, "confident_wrong": 15, "macro_f1": 0.05714285714285714, "confusion_matrix": { "0": { "0": 0, "1": 4, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 4, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 4, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 8, "accuracy": 0.42105263157894735, "wilson95": [ 0.23141891282454238, 0.6372409646574496 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42105263157894735, "brier": 0.8408033197456222, "nll": 2.3016587027539583, "ece10": 0.24117915296692166, "calibration_n": 19, "confident_wrong": 1, "macro_f1": 0.31538461538461543, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 1, "roads": 0, "noise": 0, "permits": 3, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 1, "roads": 0, "noise": 0, "permits": 3, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 1, "roads": 0, "noise": 0, "permits": 3, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 2, "roads": 0, "noise": 0, "permits": 0, "utilities": 1, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 7, "accuracy": 0.4117647058823529, "wilson95": [ 0.2161105679021481, 0.6399455735700021 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4117647058823529, "brier": 1.1122706941514346, "nll": 2.925431328606971, "ece10": 0.539471464200847, "calibration_n": 17, "confident_wrong": 8, "macro_f1": 0.22916666666666666, "confusion_matrix": { "bug": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 0, "security": 0, "style": 2, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 1, "refactor": 0, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 4, "refactor": 0, "nit": 0, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 5, "accuracy": 0.3333333333333333, "wilson95": [ 0.1517632443971473, 0.5828645226148023 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3333333333333333, "brier": 1.0158951658025008, "nll": 2.0479795364817486, "ece10": 0.4955866781649258, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.2547008547008547, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 1, "fix": 1, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 1, "refactor": 1, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 1, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 1, "__failed__": 0 }, "test": { "feature": 1, "fix": 1, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "chore": { "feature": 4, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.48886755288007655, "nll": 0.6813490422313176, "ece10": 0.11585274710889945, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.4711111111111111, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6263803998862469, "nll": 0.8404620812264718, "ece10": 0.2779733024476465, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.3333333333333333, "confusion_matrix": { "false": { "false": 0, "true": 8, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 4, "accuracy": 0.2222222222222222, "wilson95": [ 0.09000928108601694, 0.45214584316212625 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2222222222222222, "brier": 1.022875196835676, "nll": 2.2385509786062494, "ece10": 0.43694122073846065, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.09411764705882353, "confusion_matrix": { "harassment": { "harassment": 0, "threat": 3, "self_harm": 0, "spam": 0, "none": 2, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 2, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 1, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 1, "self_harm": 0, "spam": 0, "none": 3, "__failed__": 0 }, "none": { "harassment": 0, "threat": 2, "self_harm": 0, "spam": 0, "none": 4, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 7, "accuracy": 0.3181818181818182, "wilson95": [ 0.1636059318135624, 0.5268140225895723 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3181818181818182, "brier": 0.9511914861402938, "nll": 2.4810089870469456, "ece10": 0.35962616213903603, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.28376068376068375, "confusion_matrix": { "news_report": { "news_report": 2, "opinion": 0, "press_release": 3, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 2, "opinion": 2, "press_release": 1, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 1, "opinion": 0, "press_release": 2, "analysis": 0, "feature": 1, "__failed__": 0 }, "analysis": { "news_report": 2, "opinion": 1, "press_release": 1, "analysis": 0, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 0, "opinion": 1, "press_release": 2, "analysis": 0, "feature": 1, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.8968309883358458, "nll": 2.641089613100904, "ece10": 0.3703608715808369, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.37142857142857144, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 2, "force_majeure": 1, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 1, "indemnification": 2, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 8, "accuracy": 0.42105263157894735, "wilson95": [ 0.23141891282454238, 0.6372409646574496 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42105263157894735, "brier": 0.7726216654583412, "nll": 1.8306506819253021, "ece10": 0.3806142497465826, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.3688888888888889, "confusion_matrix": { "address_issue": { "address_issue": 1, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 3, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 2, "weather_delay": 0, "failed_attempt": 0, "damaged": 3, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 1, "weather_delay": 1, "failed_attempt": 0, "damaged": 1, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 3, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.5915541843512814, "nll": 0.8057775280235075, "ece10": 0.2251549364092011, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.34615384615384615, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 8, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 4, "accuracy": 0.26666666666666666, "wilson95": [ 0.10897453325692374, 0.5195043405598057 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.26666666666666666, "brier": 0.8648886784021865, "nll": 1.6391705044982794, "ece10": 0.3858505229722373, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.24, "confusion_matrix": { "invoice": { "invoice": 0, "purchase_order": 3, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 1, "purchase_order": 1, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 2, "purchase_order": 1, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 3, "receipt": 1, "contract": 0, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 7, "accuracy": 0.4666666666666667, "wilson95": [ 0.24809535365737387, 0.698830199745016 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.8391689994668375, "nll": 1.8755809341963479, "ece10": 0.42913096102860954, "calibration_n": 15, "confident_wrong": 2, "macro_f1": 0.3637362637362637, "confusion_matrix": { "meeting_request": { "meeting_request": 3, "question": 0, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 2, "question": 0, "complaint": 1, "thanks": 0, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 3, "question": 0, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 7, "accuracy": 0.3888888888888889, "wilson95": [ 0.2030524657216362, 0.613809583977621 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3888888888888889, "brier": 0.9517566556967777, "nll": 1.7488186449392078, "ece10": 0.4267698664071017, "calibration_n": 18, "confident_wrong": 4, "macro_f1": 0.26666666666666666, "confusion_matrix": { "meals": { "meals": 0, "travel": 2, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 3, "office_supplies": 0, "software": 1, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 3, "office_supplies": 0, "software": 1, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 0, "travel": 2, "office_supplies": 0, "software": 0, "client_entertainment": 1, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 8, "accuracy": 0.4444444444444444, "wilson95": [ 0.24559518127005633, 0.6628358435795723 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4444444444444444, "brier": 0.5111320172080513, "nll": 0.7043052472789654, "ece10": 0.09131253027504786, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.4375, "confusion_matrix": { "false": { "false": 5, "true": 4, "__failed__": 0 }, "true": { "false": 6, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 6, "accuracy": 0.375, "wilson95": [ 0.18481232558863633, 0.6135895945449727 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.375, "brier": 0.7716985037437722, "nll": 1.424925842257466, "ece10": 0.3528172568114101, "calibration_n": 16, "confident_wrong": 2, "macro_f1": 0.34090909090909094, "confusion_matrix": { "0": { "0": 1, "1": 2, "2": 1, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 5, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 7, "accuracy": 0.3888888888888889, "wilson95": [ 0.2030524657216362, 0.613809583977621 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3888888888888889, "brier": 0.7973273017655162, "nll": 1.885831681784932, "ece10": 0.3550040677404474, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.2571428571428571, "confusion_matrix": { "0": { "0": 0, "1": 2, "2": 3, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 6, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 10, "accuracy": 0.5, "wilson95": [ 0.2992980081982123, 0.7007019918017877 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.7394508861596207, "nll": 1.8135467347197511, "ece10": 0.234400584812598, "calibration_n": 20, "confident_wrong": 1, "macro_f1": 0.3885964912280701, "confusion_matrix": { "cheating": { "cheating": 2, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 2, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 1, "boosting": 0, "smurfing": 0, "other": 3, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 3, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 4, "accuracy": 0.19047619047619047, "wilson95": [ 0.07667565052761793, 0.40000564690204926 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.19047619047619047, "brier": 1.0355725976955983, "nll": 2.4037786791600553, "ece10": 0.5024134898286717, "calibration_n": 21, "confident_wrong": 2, "macro_f1": 0.10444444444444442, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 4, "none": 0, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 0, "punctuation": 2, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 0, "punctuation": 3, "none": 0, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 0, "punctuation": 3, "none": 0, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 0, "punctuation": 3, "none": 0, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 7, "accuracy": 0.4375, "wilson95": [ 0.23098652405492354, 0.6682144360118811 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4375, "brier": 0.7145876052619176, "nll": 0.9444347048572896, "ece10": 0.34939476558749555, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.30434782608695654, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 9, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 9, "accuracy": 0.391304347826087, "wilson95": [ 0.22157622792477952, 0.592144776086183 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.391304347826087, "brier": 0.8504152301374208, "nll": 2.136961787027052, "ece10": 0.2940399330419377, "calibration_n": 23, "confident_wrong": 1, "macro_f1": 0.275, "confusion_matrix": { "plumbing": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 2, "hvac": 1, "handyman": 1, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 2, "hvac": 0, "handyman": 3, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 1, "hvac": 1, "handyman": 2, "appliance_repair": 0, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 3, "accuracy": 0.21428571428571427, "wilson95": [ 0.07571386712693184, 0.47589230586600295 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 3, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.21428571428571427, "brier": 1.2335339968336814, "nll": 2.7847770045986224, "ece10": 0.6061299242791346, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.07058823529411765, "confusion_matrix": { "0": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 0, "2": 0, "3": 5, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.8055668202289008, "nll": 2.7296481198225493, "ece10": 0.36424994888060314, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.34285714285714286, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 0, "2": 0, "3": 3, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 0, "3": 3, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 4, "accuracy": 0.25, "wilson95": [ 0.10182067491213048, 0.49498316535508774 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.25, "brier": 0.9826064493560237, "nll": 1.9834571653414197, "ece10": 0.47291097789824743, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.1111111111111111, "confusion_matrix": { "0": { "0": 0, "1": 2, "2": 2, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.5926437905851036, "nll": 0.789472872068767, "ece10": 0.27098934496619764, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.3952569169960475, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 8, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 4, "accuracy": 0.23529411764705882, "wilson95": [ 0.09555044322080117, 0.4726179811956498 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.23529411764705882, "brier": 1.177833000821853, "nll": 2.1718031755122373, "ece10": 0.603244951925719, "calibration_n": 17, "confident_wrong": 5, "macro_f1": 0.16666666666666666, "confusion_matrix": { "politics": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 4, "__failed__": 0 }, "business": { "politics": 1, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 1, "sports": 0, "culture": 2, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 6, "accuracy": 0.35294117647058826, "wilson95": [ 0.17309720397243744, 0.5869963651478132 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35294117647058826, "brier": 0.8804193246609512, "nll": 1.7278260527911475, "ece10": 0.475408241840662, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.25555555555555554, "confusion_matrix": { "payments": { "payments": 0, "web": 1, "infra": 3, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 1, "infra": 3, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 2, "data": 1, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 2, "data": 0, "model": 0, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.5747531381777861, "nll": 0.7722267805684625, "ece10": 0.19640980643361122, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.32, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 9, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.4718196855914899, "nll": 0.6612645126390637, "ece10": 0.10390429292452288, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5151515151515151, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 6, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 5, "accuracy": 0.3125, "wilson95": [ 0.14164643854782039, 0.5559564416525933 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3125, "brier": 1.1790565961989603, "nll": 3.960605591640563, "ece10": 0.5686704231045225, "calibration_n": 16, "confident_wrong": 5, "macro_f1": 0.15873015873015872, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "2": { "0": 6, "1": 0, "2": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 5, "accuracy": 0.2777777777777778, "wilson95": [ 0.12499753379553993, 0.5087265656029746 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2777777777777778, "brier": 1.102820764328807, "nll": 2.3855391251621083, "ece10": 0.4935246346782759, "calibration_n": 18, "confident_wrong": 4, "macro_f1": 0.1977124183006536, "confusion_matrix": { "italian": { "italian": 1, "french": 0, "mexican": 0, "indian": 1, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 1, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 0, "indian": 2, "thai": 0, "middle_eastern": 1, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 2, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 1, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.4906554882820063, "nll": 0.6838082942917997, "ece10": 0.14827700394730656, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.4711111111111111, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 4, "accuracy": 0.2222222222222222, "wilson95": [ 0.09000928108601694, 0.45214584316212625 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2222222222222222, "brier": 1.1540145636802068, "nll": 3.5914192335903863, "ece10": 0.5500557675299174, "calibration_n": 18, "confident_wrong": 4, "macro_f1": 0.17058823529411765, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 1, "wrong_item": 1, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 4, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 4, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 2, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 4, "accuracy": 0.2222222222222222, "wilson95": [ 0.09000928108601694, 0.45214584316212625 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2222222222222222, "brier": 1.2160851393521621, "nll": 2.6468136253453998, "ece10": 0.5944827088296887, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.07272727272727272, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "4": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 7, "accuracy": 0.4375, "wilson95": [ 0.23098652405492354, 0.6682144360118811 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4375, "brier": 0.5437225284604171, "nll": 0.7377440596598626, "ece10": 0.11739168457171395, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.43529411764705883, "confusion_matrix": { "false": { "false": 4, "true": 6, "__failed__": 0 }, "true": { "false": 3, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 7, "accuracy": 0.4666666666666667, "wilson95": [ 0.24809535365737387, 0.698830199745016 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.4899826856325395, "nll": 0.6821079869639746, "ece10": 0.31676396732641665, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.3181818181818182, "confusion_matrix": { "false": { "false": 0, "true": 8, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 4, "accuracy": 0.23529411764705882, "wilson95": [ 0.09555044322080117, 0.4726179811956498 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.23529411764705882, "brier": 1.1512051499719806, "nll": 2.570761120050788, "ece10": 0.6058997204896318, "calibration_n": 17, "confident_wrong": 3, "macro_f1": 0.07619047619047618, "confusion_matrix": { "billing": { "billing": 0, "tech": 0, "sales": 5, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 0, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 4, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 2, "account": 0, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.5480269021234112, "nll": 0.7424238830141514, "ece10": 0.18814389998276268, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.32, "confusion_matrix": { "false": { "false": 0, "true": 8, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 7, "accuracy": 0.3684210526315789, "wilson95": [ 0.19149512745135072, 0.5896046683519693 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3684210526315789, "brier": 1.254616083633715, "nll": 5.017021223431154, "ece10": 0.6276497877479481, "calibration_n": 19, "confident_wrong": 12, "macro_f1": 0.1346153846153846, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 2, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.4777601250500152, "nll": 0.6740061554011708, "ece10": 0.04774458566306311, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.39285714285714285, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.2853793967481302, "nll": 0.44227284493689756, "ece10": 0.149126668746237, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8328173374613003, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 7, "accuracy": 0.35, "wilson95": [ 0.18119182410108206, 0.5671457233147638 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35, "brier": 1.2995336096393888, "nll": 6.878279906921075, "ece10": 0.6497928451205596, "calibration_n": 20, "confident_wrong": 13, "macro_f1": 0.12962962962962962, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 11, "accuracy": 0.55, "wilson95": [ 0.34208534245034233, 0.7418021417443759 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.6879232751382935, "nll": 1.5238046923294286, "ece10": 0.2868544821082101, "calibration_n": 20, "confident_wrong": 2, "macro_f1": 0.4390756302521009, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 3, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 4, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 12, "accuracy": 0.6, "wilson95": [ 0.3865815007622531, 0.781193467627183 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.48210044052226253, "nll": 0.6751885646239193, "ece10": 0.07128060242398468, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.595959595959596, "confusion_matrix": { "false": { "false": 5, "true": 4, "__failed__": 0 }, "true": { "false": 4, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 7, "accuracy": 0.3181818181818182, "wilson95": [ 0.1636059318135624, 0.5268140225895723 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3181818181818182, "brier": 0.9047045636342468, "nll": 1.6689642322131057, "ece10": 0.2993087389512072, "calibration_n": 22, "confident_wrong": 1, "macro_f1": 0.23636363636363636, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 1, "3": 0, "__failed__": 0 }, "1": { "0": 6, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 2, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 3, "1": 1, "2": 2, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 391, "accuracy": 0.43444444444444447, "wilson95": [ 0.4024066734317122, 0.4670394569173393 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 481, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4482758620689655, "cluster_bootstrap95": [ 0.36731533709374165, 0.49904832321098497 ], "independent_source_groups": 596, "brier": 0.6874280716411694, "nll": 1.412365069186633, "ece10": 0.18691223473545068, "calibration_n": 1073, "confident_wrong": 6 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 57, "accuracy": 0.09930313588850175, "wilson95": [ 0.07743733979034548, 0.12649655422479592 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 233, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.24893162393162394, "brier": 1.123091612365788, "nll": 3.5318945035802307, "ece10": 0.46658940228868706, "calibration_n": 936, "confident_wrong": 160 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 134, "accuracy": 0.19142857142857142, "wilson95": [ 0.16399876047261205, 0.22222665352356266 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 354, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.32536764705882354, "cluster_bootstrap95": [ 0.16045845272206305, 0.2211126961483595 ], "independent_source_groups": 650, "brier": 0.9140116664733732, "nll": 2.1880955582113533, "ece10": 0.32964077765968747, "calibration_n": 1088, "confident_wrong": 97 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 5, "accuracy": 0.0125, "wilson95": [ 0.005350750779971611, 0.028923736195967484 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 728, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.364, "brier": 1.0297343862813615, "nll": 2.616097451982849, "ece10": 0.4825570192216708, "calibration_n": 2000, "confident_wrong": 643 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 15, "accuracy": 0.5357142857142857, "wilson95": [ 0.358126892344679, 0.7046843001720462 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5357142857142857, "brier": 0.7599343923821985, "nll": 2.4489290839449533, "ece10": 0.3803822636906641, "calibration_n": 28, "confident_wrong": 7, "macro_f1": 0.46001221001220993, "confusion_matrix": { "design": { "design": 6, "inspection": 0, "purchase": 0, "work": 1, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 2, "work": 4, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 15, "accuracy": 0.5357142857142857, "wilson95": [ 0.358126892344679, 0.7046843001720462 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5357142857142857, "brier": 0.7599343923821985, "nll": 2.4489290839449533, "ece10": 0.3803822636906641, "calibration_n": 28, "confident_wrong": 7, "macro_f1": 0.46001221001221, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 4, "purchase": 2, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "design": { "work": 1, "purchase": 0, "inspection": 0, "design": 6, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 8, "accuracy": 0.2857142857142857, "wilson95": [ 0.1525399605666909, 0.47059288433295715 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2857142857142857, "brier": 1.3203939522762091, "nll": 3.5031228690260146, "ece10": 0.688009964766702, "calibration_n": 28, "confident_wrong": 17, "macro_f1": 0.16544117647058823, "confusion_matrix": { "design": { "design": 1, "inspection": 0, "purchase": 0, "work": 6, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 8, "accuracy": 0.2857142857142857, "wilson95": [ 0.1525399605666909, 0.47059288433295715 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2857142857142857, "brier": 1.3203939522762091, "nll": 3.5031228690260146, "ece10": 0.688009964766702, "calibration_n": 28, "confident_wrong": 17, "macro_f1": 0.16544117647058823, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "design": { "work": 6, "purchase": 0, "inspection": 0, "design": 1, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 8, "accuracy": 0.2, "wilson95": [ 0.10499989725437704, 0.347573063463995 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2, "brier": 1.4718259947950216, "nll": 4.583264918042576, "ece10": 0.7586389486935518, "calibration_n": 40, "confident_wrong": 27, "macro_f1": 0.06666666666666667, "confusion_matrix": { "review": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 }, "it": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 8, "accuracy": 0.2, "wilson95": [ 0.10499989725437704, 0.347573063463995 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2, "brier": 1.4718259731300793, "nll": 4.583265021730691, "ece10": 0.7586389411918235, "calibration_n": 40, "confident_wrong": 27, "macro_f1": 0.06666666666666667, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "review": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 13, "accuracy": 0.325, "wilson95": [ 0.200845022317827, 0.47982253810122333 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.325, "brier": 1.1365011804510563, "nll": 3.227567192451602, "ece10": 0.5339798418606484, "calibration_n": 40, "confident_wrong": 15, "macro_f1": 0.22090592334494774, "confusion_matrix": { "review": { "review": 0, "it": 1, "documents": 0, "procurement": 1, "maintenance": 6, "__failed__": 0 }, "it": { "review": 0, "it": 5, "documents": 0, "procurement": 0, "maintenance": 3, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 13, "accuracy": 0.325, "wilson95": [ 0.200845022317827, 0.47982253810122333 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.325, "brier": 1.1365012363889737, "nll": 3.2275674949611046, "ece10": 0.5339798356398677, "calibration_n": 40, "confident_wrong": 15, "macro_f1": 0.22090592334494774, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 3, "procurement": 0, "documents": 0, "it": 5, "review": 0, "__failed__": 0 }, "review": { "maintenance": 6, "procurement": 1, "documents": 0, "it": 1, "review": 0, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 99, "accuracy": 0.2475, "wilson95": [ 0.20774293744415318, 0.29206077119466656 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 99, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2475, "brier": 1.1377137190855142, "nll": 5.368896275754889, "ece10": 0.3828823932597223, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.09092912229159016, "confusion_matrix": { "sadness": { "sadness": 95, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 24, "__failed__": 0 }, "joy": { "sadness": 56, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 62, "__failed__": 0 }, "love": { "sadness": 25, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 12, "__failed__": 0 }, "anger": { "sadness": 49, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 15, "__failed__": 0 }, "fear": { "sadness": 39, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 13, "__failed__": 0 }, "surprise": { "sadness": 6, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 4, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 90, "accuracy": 0.3, "wilson95": [ 0.2509396637848768, 0.35411752470494157 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 90, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3, "brier": 0.8441780191098379, "nll": 2.351476120669328, "ece10": 0.11783875489670669, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 329, "accuracy": 0.8225, "wilson95": [ 0.782040489241534, 0.8568240809128447 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 329, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8225, "brier": 0.32141649454301985, "nll": 0.5022001598592751, "ece10": 0.1600615329978363, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.7159289822455615, "confusion_matrix": { "false": { "false": 287, "true": 22, "__failed__": 0 }, "true": { "false": 49, "true": 42, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 155, "accuracy": 0.3875, "wilson95": [ 0.34104299567214824, 0.43609727055306846 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 155, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3875, "brier": 0.9244860137888402, "nll": 2.477751042559112, "ece10": 0.3650474456652437, "calibration_n": 400, "confident_wrong": 51, "macro_f1": 0.3624507042253521, "confusion_matrix": { "world": { "world": 32, "sports": 0, "business": 7, "sci_tech": 64, "__failed__": 0 }, "sports": { "world": 8, "sports": 0, "business": 3, "sci_tech": 112, "__failed__": 0 }, "business": { "world": 8, "sports": 0, "business": 45, "sci_tech": 19, "__failed__": 0 }, "sci_tech": { "world": 9, "sports": 0, "business": 15, "sci_tech": 78, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 169, "accuracy": 0.4225, "wilson95": [ 0.3750554237475196, 0.47141898187429637 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 169, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4225, "brier": 0.77016276133397, "nll": 1.0719560816521778, "ece10": 0.41349664169922, "calibration_n": 400, "confident_wrong": 45, "macro_f1": 0.3457527030184591, "confusion_matrix": { "true": { "true": 153, "false": 0, "__failed__": 0 }, "false": { "true": 231, "false": 16, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 11, "accuracy": 0.03666666666666667, "wilson95": [ 0.02059530516285303, 0.06445384817189295 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.03666666666666667, "brier": 1.1359557267033145, "nll": 3.8816193801263634, "ece10": 0.3736069119198653, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 274, "accuracy": 0.685, "wilson95": [ 0.637901366761192, 0.7285790843351184 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 274, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.685, "brier": 0.43213358480924635, "nll": 0.6232418066303187, "ece10": 0.10525237609651297, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.6846136517233611, "confusion_matrix": { "false": { "false": 130, "true": 58, "__failed__": 0 }, "true": { "false": 68, "true": 144, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 200, "accuracy": 0.5, "wilson95": [ 0.451234504169709, 0.548765495830291 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 200, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.631484425395101, "nll": 0.8558047164879409, "ece10": 0.28475076476119693, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.3333333333333333, "confusion_matrix": { "false": { "false": 200, "true": 0, "__failed__": 0 }, "true": { "false": 200, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 54, "accuracy": 0.54, "wilson95": [ 0.44264860323368227, 0.6343919169097589 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 54, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.54, "brier": 0.9193877387807419, "nll": 4.732636447505614, "ece10": 0.459836051446831, "calibration_n": 100, "confident_wrong": 46, "macro_f1": 0.35064935064935066, "confusion_matrix": { "yes": { "yes": 54, "no": 0, "__failed__": 0 }, "no": { "yes": 46, "no": 0, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 6, "accuracy": 0.06, "wilson95": [ 0.027786123963188192, 0.1247681544589592 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.06, "brier": 1.137866445128069, "nll": 4.897943346704306, "ece10": 0.3431567334068334, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.008620689655172414, "confusion_matrix": { "balance": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 4, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 3, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 5, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 2, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 2, "fraud_alert": 3, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "mortgage": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 7, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 4, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "statement": { "balance": 3, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 6, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 6, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 5, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 12, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 6, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 5, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 1, "fraud_alert": 4, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 4, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "other": { "balance": 4, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 5, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 16, "accuracy": 0.16, "wilson95": [ 0.10095288488804782, 0.24420269389270244 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 81, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.405, "brier": 0.7447550505015884, "nll": 1.7939983681781693, "ece10": 0.19111847738384835, "calibration_n": 200, "confident_wrong": 7 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 3, "accuracy": 0.03, "wilson95": [ 0.010254524024038925, 0.0845193642905276 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 56, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.28, "brier": 1.0407378928406288, "nll": 2.9154656418622187, "ece10": 0.372276524113967, "calibration_n": 200, "confident_wrong": 54 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 15, "accuracy": 0.15, "wilson95": [ 0.0930598528365663, 0.23283559590832364 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.15, "brier": 1.1275662048471524, "nll": 3.3691669438885787, "ece10": 0.4354393270036279, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.11358948131100029, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 15, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 1, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 4, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 2, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 2, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "bank_statement": { "invoice": 14, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 6, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 3, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 5, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 2, "lease": 0, "research_abstract": 0, "recipe": 1, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 9, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 5, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 3, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 2, "receipt": 0, "contract": 2, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 2, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 1, "sports_recap": 0, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 139, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3475, "brier": 1.1050231034701776, "nll": 3.1546729085102196, "ece10": 0.5245148296494884, "calibration_n": 400, "confident_wrong": 124 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 21, "accuracy": 0.21, "wilson95": [ 0.141656540619153, 0.29979968834089865 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.21, "brier": 0.957950129213801, "nll": 2.995792437216981, "ece10": 0.2149692834952861, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.14215455276430886, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "science": 0, "health": 4, "sports": 0, "entertainment": 4, "world": 0, "culture": 1, "weather": 1, "__failed__": 0 }, "business": { "politics": 1, "business": 0, "technology": 1, "science": 0, "health": 1, "sports": 0, "entertainment": 7, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 9, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 5, "business": 0, "technology": 2, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 0, "culture": 0, "weather": 2, "__failed__": 0 }, "health": { "politics": 1, "business": 0, "technology": 0, "science": 0, "health": 8, "sports": 0, "entertainment": 1, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 2, "business": 0, "technology": 0, "science": 0, "health": 2, "sports": 0, "entertainment": 3, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "entertainment": { "politics": 3, "business": 0, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 6, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "world": { "politics": 1, "business": 0, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 8, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 4, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 5, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "weather": { "politics": 6, "business": 0, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 0, "culture": 0, "weather": 2, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 13, "accuracy": 0.13, "wilson95": [ 0.07757167427240512, 0.20980351440076428 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.13, "brier": 1.0801977282394553, "nll": 4.401904338765824, "ece10": 0.3466775384675613, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.0936058601962492, "confusion_matrix": { "biology": { "biology": 1, "medicine": 0, "chemistry": 0, "physics": 3, "computer_science": 0, "mathematics": 3, "earth_science": 1, "engineering": 0, "economics": 1, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 0, "chemistry": 4, "physics": 1, "computer_science": 0, "mathematics": 3, "earth_science": 0, "engineering": 0, "economics": 1, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 2, "physics": 2, "computer_science": 0, "mathematics": 2, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 0, "medicine": 0, "chemistry": 4, "physics": 5, "computer_science": 0, "mathematics": 7, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 0, "chemistry": 3, "physics": 5, "computer_science": 0, "mathematics": 6, "earth_science": 0, "engineering": 0, "economics": 1, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 3, "computer_science": 0, "mathematics": 3, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 1, "medicine": 0, "chemistry": 7, "physics": 2, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 1, "other": 0, "__failed__": 0 }, "engineering": { "biology": 1, "medicine": 0, "chemistry": 4, "physics": 0, "computer_science": 0, "mathematics": 2, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 0, "medicine": 0, "chemistry": 2, "physics": 3, "computer_science": 0, "mathematics": 1, "earth_science": 0, "engineering": 0, "economics": 2, "other": 0, "__failed__": 0 }, "other": { "biology": 0, "medicine": 0, "chemistry": 1, "physics": 6, "computer_science": 0, "mathematics": 5, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.1, "brier": 1.2061590823595048, "nll": 4.3077913780650015, "ece10": 0.5169580876049223, "calibration_n": 100, "confident_wrong": 5 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 30, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.15, "brier": 1.0648582548450685, "nll": 1.7793124562610243, "ece10": 0.5267518429580718, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 31, "accuracy": 0.31, "wilson95": [ 0.22779697212376038, 0.40626055719489407 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 31, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.31, "brier": 0.9883371489261613, "nll": 1.6742723071630448, "ece10": 0.4690833397941557, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.15776081424936386, "confusion_matrix": { "negative": { "negative": 0, "neutral": 0, "positive": 36, "__failed__": 0 }, "neutral": { "negative": 0, "neutral": 0, "positive": 33, "__failed__": 0 }, "positive": { "negative": 0, "neutral": 0, "positive": 31, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 3, "accuracy": 0.03, "wilson95": [ 0.010254524024038925, 0.0845193642905276 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 52, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.26, "brier": 0.5432058100538536, "nll": 0.7401655117993533, "ece10": 0.1384468145540211, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 105, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35, "brier": 0.9147901233492022, "nll": 2.2435125432892344, "ece10": 0.3500196246332177, "calibration_n": 300, "confident_wrong": 33 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 4, "accuracy": 0.04, "wilson95": [ 0.01566330399154762, 0.0983707143588792 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.04, "brier": 1.9078276789443165, "nll": 10.044038940779114, "ece10": 0.953960036793362, "calibration_n": 100, "confident_wrong": 96, "macro_f1": 0.0027472527472527475, "confusion_matrix": { "order_status": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 3, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 4, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 3, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 3, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 6, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 3, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 5, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 4, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 4, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 4, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 5, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 7, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 4, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 9, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 5, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 7, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 5, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 3, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.08, "brier": 1.2555045990817257, "nll": 4.690749017580534, "ece10": 0.5230622226304462, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.013745704467353952, "confusion_matrix": { "billing": { "billing": 0, "account_access": 0, "shipping": 10, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 3, "account_access": 0, "shipping": 8, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 8, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 0, "account_access": 0, "shipping": 7, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 0, "shipping": 7, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 7, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 2, "account_access": 0, "shipping": 7, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 0, "shipping": 6, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 1, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 7, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 0, "shipping": 6, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 0, "shipping": 7, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "account_access": 0, "shipping": 9, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 79, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2633333333333333, "brier": 1.2557535695720936, "nll": 3.8266077362646382, "ece10": 0.5957402006631032, "calibration_n": 300, "confident_wrong": 103 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.05, "brier": 1.3874441826062556, "nll": 4.837179000664737, "ece10": 0.6241382956048088, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.018827838827838828, "confusion_matrix": { "new_booking": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 5, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 10, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 0, "car": 4, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 0, "car": 7, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 2, "refund": 0, "hotel": 0, "car": 11, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 2, "refund": 0, "hotel": 0, "car": 5, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 0, "car": 10, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 0, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 2, "refund": 0, "hotel": 0, "car": 6, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 0, "car": 8, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 7, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 0, "car": 11, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 23, "accuracy": 0.23, "wilson95": [ 0.15843265880303448, 0.3215438302287378 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.23, "brier": 1.3136488070526309, "nll": 3.805050879565187, "ece10": 0.6574055218158243, "calibration_n": 100, "confident_wrong": 51, "macro_f1": 0.10833333333333332, "confusion_matrix": { "business": { "business": 1, "science and technology": 24, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 3, "science and technology": 22, "sports": 0, "world": 0, "__failed__": 0 }, "sports": { "business": 1, "science and technology": 24, "sports": 0, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 25, "sports": 0, "world": 0, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 3, "accuracy": 0.03, "wilson95": [ 0.010254524024038925, 0.0845193642905276 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 3, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.03, "brier": 1.1114813647240016, "nll": 5.9050273262688355, "ece10": 0.28391323727317463, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.005874029828733661, "confusion_matrix": { "refund not showing up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 2, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 2, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 2, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 2, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 2, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 2, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 2, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 2, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 2, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 2, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 21, "accuracy": 0.21, "wilson95": [ 0.141656540619153, 0.29979968834089865 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.21, "brier": 1.176690782018721, "nll": 5.456178331375282, "ece10": 0.42113476910465975, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.10819262038774236, "confusion_matrix": { "anger": { "anger": 0, "fear": 0, "joy": 0, "love": 0, "sadness": 10, "surprise": 7, "__failed__": 0 }, "fear": { "anger": 0, "fear": 0, "joy": 0, "love": 0, "sadness": 13, "surprise": 4, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 0, "love": 0, "sadness": 14, "surprise": 3, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 0, "love": 0, "sadness": 15, "surprise": 2, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 0, "joy": 0, "love": 0, "sadness": 14, "surprise": 2, "__failed__": 0 }, "surprise": { "anger": 0, "fear": 0, "joy": 0, "love": 0, "sadness": 9, "surprise": 7, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1229, "accuracy": 0.4168928086838535, "wilson95": [ 0.3992142368266082, 0.4347876881291574 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1229, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4168928086838535, "brier": 0.7586307049805119, "nll": 2.05857727926749, "ece10": 0.17119530510641026, "calibration_n": 2948, "confident_wrong": 51, "macro_f1": 0.3727847471658595, "confusion_matrix": { "alarm": { "alarm": 15, "audio": 0, "calendar": 81, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 0, "audio": 25, "calendar": 12, "cooking": 0, "datetime": 0, "email": 1, "general": 8, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 14, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 0, "audio": 0, "calendar": 376, "cooking": 2, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 6, "news": 8, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 63, "datetime": 0, "email": 0, "general": 2, "iot": 1, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 53, "cooking": 0, "datetime": 41, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 4, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 226, "cooking": 0, "datetime": 0, "email": 8, "general": 27, "iot": 0, "lists": 0, "music": 3, "news": 1, "play": 0, "qa": 0, "recommendation": 0, "social": 3, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 2, "calendar": 47, "cooking": 14, "datetime": 0, "email": 3, "general": 45, "iot": 5, "lists": 0, "music": 28, "news": 8, "play": 16, "qa": 9, "recommendation": 0, "social": 2, "takeaway": 4, "transport": 1, "weather": 3, "__failed__": 0 }, "iot": { "alarm": 18, "audio": 6, "calendar": 61, "cooking": 13, "datetime": 0, "email": 0, "general": 8, "iot": 40, "lists": 0, "music": 3, "news": 0, "play": 61, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 3, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 87, "cooking": 8, "datetime": 0, "email": 1, "general": 15, "iot": 0, "lists": 17, "music": 9, "news": 2, "play": 2, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 9, "cooking": 1, "datetime": 0, "email": 1, "general": 11, "iot": 0, "lists": 0, "music": 50, "news": 0, "play": 9, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 1, "calendar": 13, "cooking": 0, "datetime": 0, "email": 1, "general": 8, "iot": 0, "lists": 0, "music": 26, "news": 70, "play": 0, "qa": 2, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "play": { "alarm": 0, "audio": 14, "calendar": 60, "cooking": 4, "datetime": 1, "email": 4, "general": 25, "iot": 0, "lists": 0, "music": 46, "news": 1, "play": 229, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 1, "calendar": 11, "cooking": 11, "datetime": 7, "email": 2, "general": 48, "iot": 2, "lists": 1, "music": 65, "news": 44, "play": 12, "qa": 46, "recommendation": 0, "social": 0, "takeaway": 22, "transport": 2, "weather": 13, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 26, "cooking": 10, "datetime": 1, "email": 0, "general": 1, "iot": 1, "lists": 0, "music": 29, "news": 2, "play": 6, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 15, "transport": 2, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 1, "calendar": 34, "cooking": 2, "datetime": 0, "email": 3, "general": 39, "iot": 0, "lists": 1, "music": 11, "news": 1, "play": 2, "qa": 0, "recommendation": 0, "social": 10, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 14, "cooking": 1, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 3, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 38, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 56, "cooking": 1, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 2, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 18, "transport": 45, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 13, "cooking": 5, "datetime": 0, "email": 0, "general": 3, "iot": 1, "lists": 0, "music": 7, "news": 0, "play": 8, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 3, "transport": 1, "weather": 110, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 894, "accuracy": 0.30325644504748983, "wilson95": [ 0.2869282706203023, 0.32009669458110196 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 894, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.30325644504748983, "brier": 0.8651142326121983, "nll": 2.7310535502156092, "ece10": 0.18569286409653665, "calibration_n": 2948, "confident_wrong": 21, "macro_f1": 0.2505307791311926, "confusion_matrix": { "alarm": { "alarm": 14, "audio": 0, "calendar": 42, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 28, "lists": 0, "music": 1, "news": 0, "play": 4, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 4, "transport": 3, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 0, "audio": 13, "calendar": 14, "cooking": 0, "datetime": 1, "email": 0, "general": 1, "iot": 5, "lists": 0, "music": 9, "news": 0, "play": 10, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 7, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 0, "audio": 0, "calendar": 283, "cooking": 1, "datetime": 6, "email": 1, "general": 2, "iot": 13, "lists": 0, "music": 9, "news": 3, "play": 23, "qa": 0, "recommendation": 0, "social": 2, "takeaway": 30, "transport": 27, "weather": 0, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 29, "datetime": 0, "email": 0, "general": 0, "iot": 1, "lists": 0, "music": 3, "news": 0, "play": 2, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 34, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 14, "cooking": 1, "datetime": 49, "email": 0, "general": 0, "iot": 14, "lists": 0, "music": 5, "news": 4, "play": 0, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 8, "transport": 3, "weather": 1, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 158, "cooking": 0, "datetime": 1, "email": 19, "general": 7, "iot": 14, "lists": 0, "music": 24, "news": 1, "play": 7, "qa": 0, "recommendation": 0, "social": 3, "takeaway": 23, "transport": 11, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 16, "cooking": 6, "datetime": 5, "email": 3, "general": 4, "iot": 26, "lists": 1, "music": 22, "news": 4, "play": 44, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 37, "transport": 12, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 1, "calendar": 29, "cooking": 5, "datetime": 1, "email": 3, "general": 3, "iot": 23, "lists": 0, "music": 13, "news": 0, "play": 87, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 38, "transport": 11, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 40, "cooking": 2, "datetime": 1, "email": 0, "general": 6, "iot": 23, "lists": 2, "music": 24, "news": 0, "play": 9, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 29, "transport": 6, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 1, "datetime": 1, "email": 0, "general": 2, "iot": 2, "lists": 0, "music": 40, "news": 0, "play": 29, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 7, "cooking": 3, "datetime": 2, "email": 0, "general": 3, "iot": 22, "lists": 0, "music": 18, "news": 38, "play": 4, "qa": 0, "recommendation": 0, "social": 2, "takeaway": 13, "transport": 9, "weather": 3, "__failed__": 0 }, "play": { "alarm": 0, "audio": 23, "calendar": 28, "cooking": 1, "datetime": 2, "email": 1, "general": 8, "iot": 3, "lists": 0, "music": 114, "news": 1, "play": 177, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 4, "transport": 22, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 8, "cooking": 2, "datetime": 28, "email": 0, "general": 4, "iot": 17, "lists": 1, "music": 38, "news": 27, "play": 47, "qa": 10, "recommendation": 1, "social": 0, "takeaway": 71, "transport": 25, "weather": 8, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 2, "datetime": 0, "email": 0, "general": 0, "iot": 5, "lists": 0, "music": 13, "news": 1, "play": 17, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 39, "transport": 16, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 2, "calendar": 19, "cooking": 1, "datetime": 2, "email": 1, "general": 6, "iot": 9, "lists": 0, "music": 32, "news": 3, "play": 12, "qa": 0, "recommendation": 0, "social": 6, "takeaway": 7, "transport": 5, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 10, "cooking": 1, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 44, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 5, "cooking": 2, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 2, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 30, "transport": 81, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 9, "lists": 0, "music": 5, "news": 0, "play": 6, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 17, "transport": 49, "weather": 62, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 522, "accuracy": 0.44501278772378516, "wilson95": [ 0.41679834660848736, 0.47358620856407585 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 522, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.44501278772378516, "cluster_bootstrap95": [ 0.39415963306127383, 0.4875202163269507 ], "independent_source_groups": 47, "brier": 0.9811459668224994, "nll": 2.2449789120115464, "ece10": 0.46430666521945024, "calibration_n": 1173, "confident_wrong": 478, "macro_f1": 0.21803642842285678, "confusion_matrix": { "negative": { "negative": 1, "neutral": 179, "positive": 2, "__failed__": 0 }, "neutral": { "negative": 32, "neutral": 514, "positive": 44, "__failed__": 0 }, "positive": { "negative": 3, "neutral": 391, "positive": 7, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "cygnet-12b-nf4": { "label": "Cygnet 12B NF4 (local variant)", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 169, "accuracy": 0.845, "wilson95": [ 0.7883930940765792, 0.8886036307231068 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 169, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.845, "brier": 0.23270833003113334, "nll": 0.48051376208295954, "ece10": 0.08272760982959239, "calibration_n": 200, "confident_wrong": 10, "macro_f1": 0.8483936729415329, "confusion_matrix": { "world": { "world": 42, "sports": 0, "business": 4, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 5, "sports": 0, "business": 44, "sci_tech": 2, "__failed__": 0 }, "sci_tech": { "world": 1, "sports": 1, "business": 13, "sci_tech": 37, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 49, "accuracy": 0.245, "wilson95": [ 0.1905686808903718, 0.30904243556203415 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 49, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.245, "brier": 0.9977356258864796, "nll": 2.1686339120847133, "ece10": 0.32888484438880605, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.08004740357681535, "confusion_matrix": { "sadness": { "sadness": 48, "joy": 0, "love": 7, "anger": 0, "fear": 2, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 74, "joy": 0, "love": 4, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 11, "joy": 0, "love": 1, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 25, "joy": 0, "love": 2, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 19, "joy": 0, "love": 0, "anger": 1, "fear": 0, "surprise": 0, "__failed__": 0 }, "surprise": { "sadness": 4, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 187, "accuracy": 0.935, "wilson95": [ 0.8919809207009312, 0.961623645350847 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 187, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.935, "brier": 0.10097301698571791, "nll": 0.1855802467887811, "ece10": 0.034473706271514026, "calibration_n": 200, "confident_wrong": 2, "macro_f1": 0.9341288540954118, "confusion_matrix": { "false": { "false": 105, "true": 4, "__failed__": 0 }, "true": { "false": 9, "true": 82, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 178, "accuracy": 0.89, "wilson95": [ 0.839073090026985, 0.926227555398747 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 178, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.89, "brier": 0.1468639539280258, "nll": 0.24102164702714346, "ece10": 0.047701439770302834, "calibration_n": 200, "confident_wrong": 5, "macro_f1": 0.883276740237691, "confusion_matrix": { "true": { "true": 65, "false": 12, "__failed__": 0 }, "false": { "true": 10, "false": 113, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.14914039880539462, "nll": 0.2760138418052102, "ece10": 0.14204809691289494, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8666666666666666, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.20837423277802947, "nll": 0.40318466909838796, "ece10": 0.11313482708245456, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8533333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 1, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0035351718073487943, "nll": 0.04415890146475787, "ece10": 0.04280010247544942, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.003424218471266466, "nll": 0.04475753756156522, "ece10": 0.04345920906919478, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.1869950383684246, "nll": 0.3635525557402628, "ece10": 0.0657717336905758, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.8729648341786616, "confusion_matrix": { "label_000": { "label_000": 23, "label_001": 1, "label_002": 0, "label_003": 1, "__failed__": 0 }, "label_001": { "label_000": 7, "label_001": 15, "label_002": 1, "label_003": 2, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 43, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43, "brier": 0.8632104834626407, "nll": 1.9718156275784444, "ece10": 0.36781298852344035, "calibration_n": 100, "confident_wrong": 14, "macro_f1": 0.4127180992961257, "confusion_matrix": { "label_000": { "label_000": 8, "label_001": 3, "label_002": 1, "label_003": 0, "label_004": 5, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 2, "label_001": 9, "label_002": 0, "label_003": 1, "label_004": 4, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 9, "label_003": 2, "label_004": 4, "label_005": 1, "__failed__": 0 }, "label_003": { "label_000": 1, "label_001": 1, "label_002": 2, "label_003": 6, "label_004": 7, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 1, "label_001": 1, "label_002": 3, "label_003": 1, "label_004": 10, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 1, "label_002": 5, "label_003": 1, "label_004": 7, "label_005": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "label_000": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_001": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0006350057935743319, "nll": 0.02003171097925921, "ece10": 0.019791205059246007, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 82, "accuracy": 0.7387387387387387, "wilson95": [ 0.6500078016644905, 0.8114980042026286 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 82, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7387387387387387, "brier": 0.36672019483855883, "nll": 0.6646397410085435, "ece10": 0.08050261111434302, "calibration_n": 111, "confident_wrong": 3 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 65, "accuracy": 0.9027777777777778, "wilson95": [ 0.8126495420487314, 0.9521036800263443 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 65, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9027777777777778, "cluster_bootstrap95": [ 0.8055555555555556, 0.9722222222222222 ], "independent_source_groups": 36, "brier": 0.1643715912497872, "nll": 0.3080482332661464, "ece10": 0.06915311779551876, "calibration_n": 72, "confident_wrong": 4 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06096655112305643, "nll": 0.1284741400978439, "ece10": 0.1074767036839312, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.012123322627504651, "nll": 0.06727497461954442, "ece10": 0.06394967445804986, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.12104790112355501, "nll": 0.21483286609879215, "ece10": 0.1303906014603965, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.905982905982906, "confusion_matrix": { "false": { "false": 8, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0020550265027836027, "nll": 0.035326474177953204, "ece10": 0.03454975989223174, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.032784687953592045, "nll": 0.11542867311862576, "ece10": 0.10160010212086167, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.006127643561441789, "nll": 0.053301792478191216, "ece10": 0.05085679385216271, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.23512731729127323, "nll": 0.42871419919353043, "ece10": 0.17394545894872526, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8444444444444444, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 1, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.14888773853329154, "nll": 0.3139457174839497, "ece10": 0.12193624248089488, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9333333333333332, "confusion_matrix": { "feature": { "feature": 2, "fix": 0, "refactor": 1, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 2, "test": 0, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 2, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.10289063849215746, "nll": 0.19237289098236976, "ece10": 0.06524375652498093, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9403508771929825, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.49470939748534054, "nll": 0.7263959381379105, "ece10": 0.22346713693139528, "calibration_n": 16, "confident_wrong": 2, "macro_f1": 0.6, "confusion_matrix": { "false": { "false": 3, "true": 5, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.025668943818196534, "nll": 0.08275945916796756, "ece10": 0.07207617434986952, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "harassment": { "harassment": 5, "threat": 0, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 2, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 6, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 21, "accuracy": 0.9545454545454546, "wilson95": [ 0.7820195416134267, 0.991930572378736 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9545454545454546, "brier": 0.06936314822200432, "nll": 0.17512424126903603, "ece10": 0.10843052820665408, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 1, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 3, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0022369433054405426, "nll": 0.039168306071820265, "ece10": 0.03826458562932116, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.004211869521723613, "nll": 0.04578322090991751, "ece10": 0.044043026623135965, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0063373643511346524, "nll": 0.0476685881228281, "ece10": 0.045956579716757096, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.006161867109852634, "nll": 0.051455233488098696, "ece10": 0.04881529756480951, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.006581791964712278, "nll": 0.058145874698291106, "ece10": 0.055479632907204854, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "meeting_request": { "meeting_request": 3, "question": 0, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.043554518041500574, "nll": 0.09208746085870842, "ece10": 0.06697434140746306, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.017707216681608972, "nll": 0.081264912526068, "ece10": 0.07625471276961815, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.18249690645447494, "nll": 0.3195824837735312, "ece10": 0.09972288847952833, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8701298701298701, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.024849363179614174, "nll": 0.11624949261635795, "ece10": 0.10718400397637852, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 18, "accuracy": 0.9, "wilson95": [ 0.6989663547715128, 0.9721335187862319 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.17520609126733488, "nll": 0.39061574332984667, "ece10": 0.12374530702894973, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.891024531024531, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 4, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 1, "harassment": 0, "boosting": 0, "smurfing": 2, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 18, "accuracy": 0.8571428571428571, "wilson95": [ 0.6536393971937906, 0.9501898750027473 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.24888531779897635, "nll": 0.5144700048269398, "ece10": 0.08740384810446943, "calibration_n": 21, "confident_wrong": 1, "macro_f1": 0.8752747252747254, "confusion_matrix": { "spelling": { "spelling": 3, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 1, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 3, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 4, "punctuation": 0, "none": 0, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 3, "none": 2, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.034820802377629745, "nll": 0.10895252863207572, "ece10": 0.09789977517267402, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.031149154699167042, "nll": 0.09146854865983015, "ece10": 0.06493941448065925, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 13, "accuracy": 0.9285714285714286, "wilson95": [ 0.6853129557584889, 0.9872777847521093 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.1232064593223224, "nll": 0.29554709014067615, "ece10": 0.15467195425649624, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.9047619047619048, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.24535241042112033, "nll": 0.4308168238874173, "ece10": 0.1980643208213426, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8055555555555555, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.3419008567376916, "nll": 0.6804283310018507, "ece10": 0.17499182740694158, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8041666666666667, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 1, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.29651280879984265, "nll": 0.4438239238981137, "ece10": 0.13493552639491813, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7424242424242424, "confusion_matrix": { "false": { "false": 4, "true": 4, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04549933213565547, "nll": 0.12285836218708415, "ece10": 0.08180825822247043, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.08956634784196708, "nll": 0.21142461766344905, "ece10": 0.06867421943163972, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "payments": { "payments": 3, "web": 0, "infra": 1, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.019647075439593112, "nll": 0.07322648465630927, "ece10": 0.06717931709104724, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.22441459936758343, "nll": 0.3342430251359749, "ece10": 0.13636024483341866, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8056680161943319, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.1449011758593253, "nll": 0.22375261655076972, "ece10": 0.0864061963059832, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8771043771043772, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.0783591830070604, "nll": 0.1613208150678679, "ece10": 0.10476553397533564, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9259259259259259, "confusion_matrix": { "italian": { "italian": 1, "french": 1, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 4, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.2130748427439425, "nll": 0.37370966622332863, "ece10": 0.08822247891693177, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.8785714285714286, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06514932009450075, "nll": 0.1917196379816418, "ece10": 0.15186481646361155, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 4, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.34313652409972073, "nll": 0.6191596072957224, "ece10": 0.14667824221962572, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7061904761904761, "confusion_matrix": { "0": { "0": 1, "1": 1, "2": 1, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 1, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.4486557038059259, "nll": 0.6267153529138139, "ece10": 0.23991997134131943, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.6190476190476191, "confusion_matrix": { "false": { "false": 4, "true": 6, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 12, "accuracy": 0.8, "wilson95": [ 0.5481455128483065, 0.9295245065301844 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.2566236514951376, "nll": 0.3757287882793933, "ece10": 0.16451345959860053, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7963800904977376, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.08858921214485842, "nll": 0.2121696979119449, "ece10": 0.14718025958244874, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 4, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06775078143003725, "nll": 0.16173736090274615, "ece10": 0.13898822358119317, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 16, "accuracy": 0.8421052631578947, "wilson95": [ 0.6243452472967093, 0.9447952836146585 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8421052631578947, "brier": 0.24029077375467534, "nll": 0.4696220627255461, "ece10": 0.18851515214549594, "calibration_n": 19, "confident_wrong": 1, "macro_f1": 0.8365079365079364, "confusion_matrix": { "0": { "0": 6, "1": 0, "2": 0, "3": 1, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.2071025360321179, "nll": 0.3656519841739518, "ece10": 0.15522504526988432, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8785714285714286, "confusion_matrix": { "false": { "false": 9, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.039799858740244914, "nll": 0.09105035296426792, "ece10": 0.0756640023844796, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9442724458204335, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 18, "accuracy": 0.9, "wilson95": [ 0.6989663547715128, 0.9721335187862319 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.18980362175422666, "nll": 0.3601556408355164, "ece10": 0.12763845730962922, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8666666666666667, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 20, "accuracy": 1.0, "wilson95": [ 0.8388748419471806, 1.0 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.027825349679926697, "nll": 0.08474508585520212, "ece10": 0.06940409848498154, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 14, "accuracy": 0.7, "wilson95": [ 0.4810271816464765, 0.8545227551323956 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.39647838027926874, "nll": 0.5508815433498483, "ece10": 0.17789227554473488, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.6875, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 21, "accuracy": 0.9545454545454546, "wilson95": [ 0.7820195416134267, 0.991930572378736 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9545454545454546, "brier": 0.11130720487396445, "nll": 0.2609366555576974, "ece10": 0.13716040963751722, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9545454545454546, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 5, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 602, "accuracy": 0.6688888888888889, "wilson95": [ 0.6374819506885636, 0.698860221971507 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 765, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7129543336439889, "cluster_bootstrap95": [ 0.6275310160092734, 0.7188314094380855 ], "independent_source_groups": 596, "brier": 0.434243996267245, "nll": 0.7915790725803337, "ece10": 0.13473068843118263, "calibration_n": 1073, "confident_wrong": 84 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 449, "accuracy": 0.7822299651567944, "wilson95": [ 0.7466495175684716, 0.8140579136034723 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 800, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8547008547008547, "brier": 0.2322108531775764, "nll": 0.49158018094191525, "ece10": 0.02991795611224385, "calibration_n": 936, "confident_wrong": 24 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 288, "accuracy": 0.4114285714285714, "wilson95": [ 0.3755543250956627, 0.44826963632916533 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 610, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5606617647058824, "cluster_bootstrap95": [ 0.37606837606837606, 0.44852035059439854 ], "independent_source_groups": 650, "brier": 0.5483519731162891, "nll": 1.03844722727638, "ece10": 0.11267920180647915, "calibration_n": 1088, "confident_wrong": 12 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 79, "accuracy": 0.1975, "wilson95": [ 0.1614426641926397, 0.23931227387960974 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1364, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.682, "brier": 0.4564148531098229, "nll": 0.8202256472798756, "ece10": 0.1242981317834849, "calibration_n": 2000, "confident_wrong": 90 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.03559019841010701, "nll": 0.09976505867722683, "ece10": 0.055370426625698334, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 1, "inspection": 6, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.029678043546686936, "nll": 0.09527318604249417, "ece10": 0.05644341115541342, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 0, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.003649954854340378, "nll": 0.04857363194917636, "ece10": 0.047189387342372574, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.006541284407361263, "nll": 0.05840022237144136, "ece10": 0.055936730098687465, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04918504288914871, "nll": 0.150607751845791, "ece10": 0.1098703459407957, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "review": { "review": 8, "it": 0, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04270924080807027, "nll": 0.1366795723485026, "ece10": 0.10577531042753749, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 8, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.035570524899625054, "nll": 0.13364180864649572, "ece10": 0.11809768340607968, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "review": { "review": 8, "it": 0, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.06834694981020234, "nll": 0.17505510937488458, "ece10": 0.08486133054584906, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9499019607843138, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 7, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 7, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 8, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 107, "accuracy": 0.2675, "wilson95": [ 0.22648238411831478, 0.31294083274713314 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 107, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2675, "brier": 0.9665603997690405, "nll": 2.1126986447722502, "ece10": 0.3047226543114416, "calibration_n": 400, "confident_wrong": 2, "macro_f1": 0.09206966244441618, "confusion_matrix": { "sadness": { "sadness": 103, "joy": 0, "love": 15, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 110, "joy": 0, "love": 5, "anger": 1, "fear": 2, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 35, "joy": 0, "love": 2, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 57, "joy": 0, "love": 6, "anger": 1, "fear": 0, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 45, "joy": 0, "love": 6, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 }, "surprise": { "sadness": 10, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 271, "accuracy": 0.9033333333333333, "wilson95": [ 0.8646182980304887, 0.9318497051817106 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 271, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9033333333333333, "brier": 0.1606183583128023, "nll": 0.49648620288492606, "ece10": 0.13704031822210108, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 371, "accuracy": 0.9275, "wilson95": [ 0.8978175067464532, 0.9490494815977232 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 371, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9275, "brier": 0.10223655766919347, "nll": 0.17635981026099384, "ece10": 0.03820822929429538, "calibration_n": 400, "confident_wrong": 5, "macro_f1": 0.9016740834922653, "confusion_matrix": { "false": { "false": 288, "true": 21, "__failed__": 0 }, "true": { "false": 8, "true": 83, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 338, "accuracy": 0.845, "wilson95": [ 0.8062691036268745, 0.8771674132824607 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 338, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.845, "brier": 0.2655483247532818, "nll": 0.6066848175370663, "ece10": 0.09704240582863419, "calibration_n": 400, "confident_wrong": 35, "macro_f1": 0.8306068392864334, "confusion_matrix": { "world": { "world": 86, "sports": 5, "business": 10, "sci_tech": 2, "__failed__": 0 }, "sports": { "world": 1, "sports": 122, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 5, "sports": 1, "business": 63, "sci_tech": 3, "__failed__": 0 }, "sci_tech": { "world": 10, "sports": 0, "business": 25, "sci_tech": 67, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 362, "accuracy": 0.905, "wilson95": [ 0.8722916530824945, 0.9300033885067254 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 362, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.905, "brier": 0.1399946704935861, "nll": 0.23618107755037276, "ece10": 0.03223640115385712, "calibration_n": 400, "confident_wrong": 11, "macro_f1": 0.8983930051605657, "confusion_matrix": { "true": { "true": 130, "false": 23, "__failed__": 0 }, "false": { "true": 15, "false": 232, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 186, "accuracy": 0.62, "wilson95": [ 0.5638844388564824, 0.6730812480496265 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 186, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.62, "brier": 0.4957191746913024, "nll": 1.3113742108891537, "ece10": 0.10084333707767866, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 380, "accuracy": 0.95, "wilson95": [ 0.9240363649362803, 0.9674025701628527 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 380, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.07717911401186585, "nll": 0.15135331522163567, "ece10": 0.04087504859562892, "calibration_n": 400, "confident_wrong": 5, "macro_f1": 0.9499198717948718, "confusion_matrix": { "false": { "false": 182, "true": 6, "__failed__": 0 }, "true": { "false": 14, "true": 198, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 292, "accuracy": 0.73, "wilson95": [ 0.684457128728908, 0.7711672158773156 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 292, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.73, "brier": 0.4265395784245569, "nll": 0.669229208979956, "ece10": 0.18199473015668996, "calibration_n": 400, "confident_wrong": 52, "macro_f1": 0.7142176708740177, "confusion_matrix": { "false": { "false": 193, "true": 7, "__failed__": 0 }, "true": { "false": 101, "true": 99, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 77, "accuracy": 0.77, "wilson95": [ 0.6784561697712622, 0.8415673411969654 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 77, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.77, "brier": 0.3991618174482803, "nll": 0.7634665234377199, "ece10": 0.178528786088058, "calibration_n": 100, "confident_wrong": 18, "macro_f1": 0.7488808821923791, "confusion_matrix": { "yes": { "yes": 53, "no": 1, "__failed__": 0 }, "no": { "yes": 22, "no": 24, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 45, "accuracy": 0.45, "wilson95": [ 0.3561453797951198, 0.5475539700255787 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 45, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.7805643727384464, "nll": 1.9723998015402122, "ece10": 0.22449205689159785, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.4845624455309053, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 2, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 2, "dispute_charge": 2, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 6, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 4, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 6, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 4, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 8, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 3, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 4, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 4, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 2, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 3, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 4, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 44, "accuracy": 0.44, "wilson95": [ 0.34672026630160796, 0.5377189534832302 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 138, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.69, "brier": 0.3985410701174689, "nll": 0.6608636200416358, "ece10": 0.1652128074352477, "calibration_n": 200, "confident_wrong": 2 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 35, "accuracy": 0.35, "wilson95": [ 0.26364248243097527, 0.44745556703112027 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 116, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.6095412690883556, "nll": 1.217955304904192, "ece10": 0.20449684036741933, "calibration_n": 200, "confident_wrong": 9 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 76, "accuracy": 0.76, "wilson95": [ 0.6676766365018553, 0.8330867444305122 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 76, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.3636006040626198, "nll": 0.9202913602619234, "ece10": 0.121143534947027, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.7339913551627548, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 13, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 5, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 10, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 7, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 6, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 3, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 196, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.7986534816299227, "nll": 1.579318598135379, "ece10": 0.34616916094850736, "calibration_n": 400, "confident_wrong": 73 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 61, "accuracy": 0.61, "wilson95": [ 0.5120301704584073, 0.6998312599360558 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 61, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.5387469797875684, "nll": 1.2800485258557746, "ece10": 0.14106305711802053, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6028971028971031, "confusion_matrix": { "politics": { "politics": 5, "business": 1, "technology": 0, "science": 1, "health": 2, "sports": 0, "entertainment": 1, "world": 3, "culture": 0, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 3, "technology": 6, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 1, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 1, "technology": 1, "science": 3, "health": 1, "sports": 0, "entertainment": 0, "world": 2, "culture": 1, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 1, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 5, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 2, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 3, "world": 0, "culture": 3, "weather": 0, "__failed__": 0 }, "world": { "politics": 0, "business": 1, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 6, "culture": 1, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 8, "weather": 1, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 6, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 37, "accuracy": 0.37, "wilson95": [ 0.28182360534324524, 0.4677947041905709 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.37, "brier": 0.8801503666587804, "nll": 2.1413321196652926, "ece10": 0.2892256132233268, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.335003324656938, "confusion_matrix": { "biology": { "biology": 5, "medicine": 1, "chemistry": 1, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 9, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 1, "medicine": 1, "chemistry": 3, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 4, "medicine": 4, "chemistry": 0, "physics": 3, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 1, "economics": 2, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 2, "chemistry": 1, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 4, "engineering": 5, "economics": 2, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 2, "physics": 1, "computer_science": 0, "mathematics": 1, "earth_science": 2, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 1, "medicine": 3, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 6, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 5, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 4, "other": 0, "__failed__": 0 }, "other": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 2, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 3, "economics": 3, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 17, "accuracy": 0.17, "wilson95": [ 0.10893566114059931, 0.25548004767601123 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 81, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.405, "brier": 0.6834607771873473, "nll": 1.4881646471385936, "ece10": 0.32648425908728124, "calibration_n": 100, "confident_wrong": 24 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 29, "accuracy": 0.29, "wilson95": [ 0.2101483574912227, 0.3853889117557111 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 117, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.585, "brier": 0.33624793536285885, "nll": 0.6139507453231215, "ece10": 0.11943593643213055, "calibration_n": 100, "confident_wrong": 6 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 87, "accuracy": 0.87, "wilson95": [ 0.7901964855992357, 0.9224283257275948 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 87, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.21277519560907823, "nll": 0.39329110273868445, "ece10": 0.07591801245394741, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.8731429197960638, "confusion_matrix": { "negative": { "negative": 32, "neutral": 4, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 5, "neutral": 28, "positive": 0, "__failed__": 0 }, "positive": { "negative": 1, "neutral": 3, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 23, "accuracy": 0.23, "wilson95": [ 0.15843265880303448, 0.3215438302287378 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 109, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.545, "brier": 0.2220591939545114, "nll": 0.3555300658998693, "ece10": 0.07069003748129539, "calibration_n": 100, "confident_wrong": 3 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 51, "accuracy": 0.51, "wilson95": [ 0.4134801193402717, 0.6057800106955886 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 240, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.3013307831597731, "nll": 0.5985106401682941, "ece10": 0.06024029827936185, "calibration_n": 300, "confident_wrong": 8 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "order_status": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "refund_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 6 }, "shipping_delay": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "pricing_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "speak_to_human": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 7 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 9 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 7 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "promo_code": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "partner_inquiry": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 33, "accuracy": 0.33, "wilson95": [ 0.24563122734128034, 0.4269465620490947 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 33, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.33, "brier": 0.9545269494212186, "nll": 2.243942739354735, "ece10": 0.3615458750948402, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.29748712945114747, "confusion_matrix": { "billing": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 1, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 6, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 1, "shipping": 3, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 3, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 3, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 4, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 1, "onboarding": 2, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 6, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "compliance": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 3, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 5, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 1, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "other": { "billing": 4, "account_access": 0, "shipping": 1, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 7, "accuracy": 0.07, "wilson95": [ 0.03431926106727268, 0.137495147390735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 135, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.7752379504362652, "nll": 1.7428939112464097, "ece10": 0.2785318455603234, "calibration_n": 300, "confident_wrong": 4 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 59, "accuracy": 0.59, "wilson95": [ 0.4920143005341078, 0.6813268697886348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 59, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.59, "brier": 0.5769301531050393, "nll": 1.4364131052525386, "ece10": 0.1667975466331581, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5551652824872638, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 5, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 1, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 2, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 9, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 4, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 1, "loyalty": 4, "disruption": 1, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 5, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 3, "car": 1, "visa_docs": 4, "loyalty": 1, "disruption": 1, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.1982048717853178, "nll": 0.3744555980003794, "ece10": 0.056745594270642304, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.872278727832178, "confusion_matrix": { "business": { "business": 23, "science and technology": 1, "sports": 0, "world": 1, "__failed__": 0 }, "science and technology": { "business": 5, "science and technology": 15, "sports": 1, "world": 4, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "refund not showing up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "activating a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 45, "accuracy": 0.45, "wilson95": [ 0.3561453797951198, 0.5475539700255787 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 45, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.7845266492084129, "nll": 1.7314118181034541, "ece10": 0.31339836573024976, "calibration_n": 100, "confident_wrong": 9, "macro_f1": 0.4307594022764301, "confusion_matrix": { "anger": { "anger": 8, "fear": 4, "joy": 0, "love": 2, "sadness": 3, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 1, "fear": 11, "joy": 0, "love": 0, "sadness": 4, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 9, "love": 1, "sadness": 5, "surprise": 1, "__failed__": 0 }, "love": { "anger": 1, "fear": 1, "joy": 2, "love": 6, "sadness": 7, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 1, "fear": 1, "joy": 3, "love": 1, "sadness": 10, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 0, "joy": 4, "love": 3, "sadness": 7, "surprise": 1, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2272, "accuracy": 0.7706919945725916, "wilson95": [ 0.7551703102658747, 0.7855091341639175 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2272, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7706919945725916, "brier": 0.3048690981408081, "nll": 0.8496964840259477, "ece10": 0.07416737139739993, "calibration_n": 2948, "confident_wrong": 2, "macro_f1": 0.7685755378867217, "confusion_matrix": { "alarm": { "alarm": 85, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 4, "audio": 46, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 1, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 1, "audio": 0, "calendar": 327, "cooking": 2, "datetime": 5, "email": 2, "general": 1, "iot": 8, "lists": 25, "music": 0, "news": 3, "play": 0, "qa": 9, "recommendation": 15, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 56, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 7, "music": 0, "news": 0, "play": 1, "qa": 6, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 68, "email": 0, "general": 0, "iot": 4, "lists": 4, "music": 0, "news": 0, "play": 0, "qa": 23, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 1, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 0, "email": 212, "general": 5, "iot": 3, "lists": 22, "music": 0, "news": 0, "play": 0, "qa": 15, "recommendation": 0, "social": 5, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 9, "cooking": 3, "datetime": 1, "email": 0, "general": 73, "iot": 4, "lists": 15, "music": 0, "news": 0, "play": 6, "qa": 58, "recommendation": 12, "social": 0, "takeaway": 0, "transport": 2, "weather": 3, "__failed__": 0 }, "iot": { "alarm": 2, "audio": 2, "calendar": 2, "cooking": 3, "datetime": 0, "email": 0, "general": 4, "iot": 186, "lists": 5, "music": 1, "news": 0, "play": 1, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 7, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 2, "general": 3, "iot": 0, "lists": 130, "music": 2, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 3, "iot": 0, "lists": 6, "music": 54, "news": 0, "play": 7, "qa": 9, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 7, "lists": 6, "music": 0, "news": 83, "play": 1, "qa": 21, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 3, "calendar": 0, "cooking": 0, "datetime": 3, "email": 1, "general": 5, "iot": 11, "lists": 27, "music": 15, "news": 0, "play": 307, "qa": 12, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 0, "email": 0, "general": 4, "iot": 11, "lists": 18, "music": 0, "news": 0, "play": 0, "qa": 244, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 2, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 10, "lists": 3, "music": 0, "news": 2, "play": 4, "qa": 11, "recommendation": 64, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 4, "general": 14, "iot": 0, "lists": 6, "music": 0, "news": 0, "play": 0, "qa": 5, "recommendation": 0, "social": 74, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 3, "music": 0, "news": 0, "play": 0, "qa": 10, "recommendation": 2, "social": 0, "takeaway": 40, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 1, "datetime": 2, "email": 0, "general": 2, "iot": 3, "lists": 4, "music": 0, "news": 0, "play": 0, "qa": 5, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 99, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 3, "iot": 9, "lists": 6, "music": 0, "news": 0, "play": 0, "qa": 9, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 124, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2013, "accuracy": 0.6828358208955224, "wilson95": [ 0.6658080870797453, 0.6993876780175706 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2013, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6828358208955224, "brier": 0.41259509593994326, "nll": 1.1423336483792523, "ece10": 0.07069086578156145, "calibration_n": 2948, "confident_wrong": 7, "macro_f1": 0.6774773926259859, "confusion_matrix": { "alarm": { "alarm": 81, "audio": 0, "calendar": 6, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 7, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 3, "audio": 35, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 5, "iot": 2, "lists": 10, "music": 0, "news": 0, "play": 1, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "calendar": { "alarm": 2, "audio": 0, "calendar": 336, "cooking": 7, "datetime": 5, "email": 2, "general": 4, "iot": 1, "lists": 20, "music": 0, "news": 1, "play": 0, "qa": 11, "recommendation": 10, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 60, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 4, "music": 0, "news": 0, "play": 0, "qa": 6, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 1, "audio": 0, "calendar": 1, "cooking": 4, "datetime": 54, "email": 0, "general": 0, "iot": 8, "lists": 6, "music": 0, "news": 0, "play": 0, "qa": 25, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "email": { "alarm": 1, "audio": 0, "calendar": 3, "cooking": 5, "datetime": 4, "email": 200, "general": 8, "iot": 0, "lists": 19, "music": 0, "news": 0, "play": 0, "qa": 24, "recommendation": 1, "social": 2, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "general": { "alarm": 0, "audio": 2, "calendar": 10, "cooking": 4, "datetime": 4, "email": 1, "general": 60, "iot": 1, "lists": 13, "music": 0, "news": 2, "play": 6, "qa": 70, "recommendation": 9, "social": 0, "takeaway": 0, "transport": 3, "weather": 2, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 6, "calendar": 4, "cooking": 1, "datetime": 1, "email": 0, "general": 7, "iot": 165, "lists": 17, "music": 0, "news": 0, "play": 0, "qa": 9, "recommendation": 1, "social": 0, "takeaway": 3, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 1, "audio": 3, "calendar": 3, "cooking": 1, "datetime": 0, "email": 0, "general": 6, "iot": 2, "lists": 116, "music": 3, "news": 0, "play": 0, "qa": 4, "recommendation": 0, "social": 1, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 0, "email": 0, "general": 3, "iot": 1, "lists": 6, "music": 44, "news": 0, "play": 9, "qa": 15, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 8, "datetime": 1, "email": 0, "general": 0, "iot": 3, "lists": 3, "music": 0, "news": 69, "play": 0, "qa": 33, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "play": { "alarm": 6, "audio": 9, "calendar": 5, "cooking": 1, "datetime": 4, "email": 2, "general": 13, "iot": 10, "lists": 37, "music": 40, "news": 2, "play": 198, "qa": 58, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 2, "calendar": 3, "cooking": 1, "datetime": 0, "email": 0, "general": 7, "iot": 2, "lists": 25, "music": 0, "news": 1, "play": 0, "qa": 241, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 1, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 4, "datetime": 0, "email": 0, "general": 0, "iot": 1, "lists": 7, "music": 0, "news": 1, "play": 2, "qa": 12, "recommendation": 66, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 2, "datetime": 0, "email": 5, "general": 13, "iot": 0, "lists": 10, "music": 0, "news": 2, "play": 0, "qa": 6, "recommendation": 0, "social": 63, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 4, "datetime": 0, "email": 0, "general": 1, "iot": 3, "lists": 9, "music": 0, "news": 0, "play": 0, "qa": 7, "recommendation": 1, "social": 0, "takeaway": 31, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 2, "datetime": 2, "email": 0, "general": 1, "iot": 2, "lists": 11, "music": 0, "news": 0, "play": 0, "qa": 8, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 89, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 1, "datetime": 0, "email": 0, "general": 4, "iot": 9, "lists": 8, "music": 0, "news": 1, "play": 0, "qa": 19, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 105, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 758, "accuracy": 0.6462063086104007, "wilson95": [ 0.6184068384724695, 0.6730512822702501 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 758, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6462063086104007, "cluster_bootstrap95": [ 0.605070831230625, 0.6853523202499119 ], "independent_source_groups": 47, "brier": 0.522312068820623, "nll": 0.943081206105364, "ece10": 0.16270713640015133, "calibration_n": 1173, "confident_wrong": 90, "macro_f1": 0.6313445009672817, "confusion_matrix": { "negative": { "negative": 138, "neutral": 20, "positive": 24, "__failed__": 0 }, "neutral": { "negative": 159, "neutral": 300, "positive": 131, "__failed__": 0 }, "positive": { "negative": 38, "neutral": 43, "positive": 320, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "decider-08b": { "label": "Decider 0.8B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 174, "accuracy": 0.87, "wilson95": [ 0.8163364812775159, 0.9097179772033067 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 174, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.17354520372985796, "nll": 0.30791982864814565, "ece10": 0.02875106714134566, "calibration_n": 200, "confident_wrong": 3, "macro_f1": 0.8717131857368294, "confusion_matrix": { "world": { "world": 45, "sports": 0, "business": 1, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 1, "sports": 46, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 4, "sports": 0, "business": 37, "sci_tech": 10, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 1, "business": 3, "sci_tech": 46, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 150, "accuracy": 0.75, "wilson95": [ 0.6856590168795417, 0.8049183199318249 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 150, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3461151866351995, "nll": 0.6386917139795061, "ece10": 0.07763249896473516, "calibration_n": 200, "confident_wrong": 8, "macro_f1": 0.5644851228101014, "confusion_matrix": { "sadness": { "sadness": 47, "joy": 2, "love": 1, "anger": 6, "fear": 1, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 6, "joy": 66, "love": 4, "anger": 2, "fear": 1, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 0, "joy": 8, "love": 4, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 3, "joy": 3, "love": 2, "anger": 20, "fear": 0, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 2, "joy": 2, "love": 0, "anger": 2, "fear": 13, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 196, "accuracy": 0.98, "wilson95": [ 0.9497129130941735, 0.9921955735836505 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 196, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.98, "brier": 0.037584782100000005, "nll": 0.06454853168894088, "ece10": 0.0374795, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.9797550359348112, "confusion_matrix": { "false": { "false": 109, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 87, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": 0.9, "development": { "n": 92, "correct": 92, "wilson95": [ 0.9599186107143782, 1.0 ] }, "test_coverage": 0.93, "accepted_n": 186, "accepted_correct": 186, "accepted_accuracy": 1.0, "accepted_wilson95": [ 0.9797649109706725, 0.9999999999999998 ], "errors_per_1000_all_test_cases": 0.0, "note": "Selected on development only, applied unchanged to test. An empirical pilot estimate, not a production guarantee." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 184, "accuracy": 0.92, "wilson95": [ 0.874010511732856, 0.9501594141102401 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 184, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.92, "brier": 0.10890616509999995, "nll": 0.1706932691451854, "ece10": 0.0275665, "calibration_n": 200, "confident_wrong": 5, "macro_f1": 0.9137094164599289, "confusion_matrix": { "true": { "true": 65, "false": 12, "__failed__": 0 }, "false": { "true": 4, "false": 119, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": 0.9, "development": { "n": 79, "correct": 79, "wilson95": [ 0.9536287883460772, 1.0 ] }, "test_coverage": 0.87, "accepted_n": 174, "accepted_correct": 169, "accepted_accuracy": 0.9712643678160919, "accepted_wilson95": [ 0.9345049632382257, 0.9876647101749375 ], "errors_per_1000_all_test_cases": 25.0, "note": "Selected on development only, applied unchanged to test. An empirical pilot estimate, not a production guarantee." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 6, "accuracy": 0.5, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.5784319154933956, "nll": 1.0407484026647758, "ece10": 0.3035379378534627, "calibration_n": 12, "confident_wrong": 1, "macro_f1": 0.5266666666666666, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 1, "procurement": 1, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 1, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 2, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 5, "accuracy": 0.4166666666666667, "wilson95": [ 0.19326031365875665, 0.6804886874504503 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4166666666666667, "brier": 0.7339363243981915, "nll": 1.40271489439442, "ece10": 0.2868033693865337, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.36190476190476184, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 1, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 1, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.05560378875020253, "nll": 0.10288233074919818, "ece10": 0.08324691044030241, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.20037951607776308, "nll": 0.3525245699713362, "ece10": 0.1883293170983765, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9142857142857143, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 2, "work": 1, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 81, "accuracy": 0.81, "wilson95": [ 0.7222115462093562, 0.8748524849023126 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 81, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.81, "brier": 0.28701547968157126, "nll": 0.5511622504417195, "ece10": 0.09709680813276808, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.7700289816755392, "confusion_matrix": { "label_000": { "label_000": 25, "label_001": 0, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 15, "label_001": 6, "label_002": 1, "label_003": 3, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 61, "accuracy": 0.61, "wilson95": [ 0.5120301704584073, 0.6998312599360558 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 61, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.49067320333635245, "nll": 0.9680137100973712, "ece10": 0.06567568269077906, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5991302240644346, "confusion_matrix": { "label_000": { "label_000": 13, "label_001": 2, "label_002": 1, "label_003": 0, "label_004": 1, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 13, "label_002": 0, "label_003": 0, "label_004": 4, "label_005": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 13, "label_003": 2, "label_004": 2, "label_005": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 8, "label_003": 6, "label_004": 3, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 1, "label_001": 2, "label_002": 2, "label_003": 0, "label_004": 11, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 4, "label_002": 4, "label_003": 0, "label_004": 2, "label_005": 5, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 78, "accuracy": 0.78, "wilson95": [ 0.6892964648501421, 0.8499871761539459 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 78, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.78, "brier": 0.30579955385731017, "nll": 0.8263454672371776, "ece10": 0.05800209140943356, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.757473544973545, "confusion_matrix": { "label_000": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 1, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 1, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 1, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 2, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 2, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 1, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 1, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0014715105900622737, "nll": 0.012550594855366859, "ece10": 0.01211034461099921, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 43, "accuracy": 0.38738738738738737, "wilson95": [ 0.30197741698369296, 0.4803311651544585 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 43, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.38738738738738737, "brier": 0.8619594343668812, "nll": 1.6255380034361184, "ece10": 0.33653381728117604, "calibration_n": 111, "confident_wrong": 20 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 61, "accuracy": 0.8472222222222222, "wilson95": [ 0.7467771063249946, 0.912492912705243 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 61, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8472222222222222, "cluster_bootstrap95": [ 0.75, 0.9305555555555556 ], "independent_source_groups": 36, "brier": 0.22837943708589853, "nll": 0.3690963552175295, "ece10": 0.0660233343460903, "calibration_n": 72, "confident_wrong": 3 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 11, "accuracy": 0.5789473684210527, "wilson95": [ 0.3627590353425504, 0.7685810871754576 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5789473684210527, "brier": 0.40758531578947377, "nll": 0.5625276257395776, "ece10": 0.23560000000000003, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.5476190476190476, "confusion_matrix": { "false": { "false": 3, "true": 8, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.35024394875, "nll": 0.46692799452055417, "ece10": 0.21130625, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5636363636363637, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 11, "accuracy": 0.5, "wilson95": [ 0.3072210627372502, 0.6927789372627497 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.64616467, "nll": 0.9615963158295062, "ece10": 0.35189545454545457, "calibration_n": 22, "confident_wrong": 2, "macro_f1": 0.39950372208436724, "confusion_matrix": { "false": { "false": 10, "true": 0, "__failed__": 0 }, "true": { "false": 11, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.13350894922576861, "nll": 0.17719074072468427, "ece10": 0.10215667255537922, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.886904761904762, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 3, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 1, "feature_request": 0, "praise": 0, "pricing_billing": 3, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.3416420166339065, "nll": 0.8289427760443342, "ece10": 0.27571484264209284, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8361904761904763, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 2, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04744553023168446, "nll": 0.14277581501136538, "ece10": 0.12204666493841401, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.3613593944461852, "nll": 0.6481795751963271, "ece10": 0.14227062820045366, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7027777777777778, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 1, "security": 0, "style": 2, "refactor": 1, "nit": 0, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.21944265747707495, "nll": 0.3653957439058814, "ece10": 0.10020783558467836, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8634920634920634, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 2, "test": 0, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 1, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 3, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.4881892329411765, "nll": 0.6742957827266172, "ece10": 0.2560352941176471, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5641025641025641, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 6, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.4074691375, "nll": 0.5691917110232846, "ece10": 0.16285000000000002, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 5, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.5382385262200965, "nll": 1.252346631699859, "ece10": 0.24552154354943764, "calibration_n": 18, "confident_wrong": 2, "macro_f1": 0.6166666666666666, "confusion_matrix": { "harassment": { "harassment": 3, "threat": 0, "self_harm": 0, "spam": 0, "none": 2, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 1, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 5, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 15, "accuracy": 0.6818181818181818, "wilson95": [ 0.47318597741042745, 0.8363940681864375 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6818181818181818, "brier": 0.39561441554783466, "nll": 0.6286839274739106, "ece10": 0.17461214267921235, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.6287581699346405, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 1, "opinion": 4, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 2, "opinion": 0, "press_release": 0, "analysis": 2, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 4, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.13536343442108445, "nll": 0.26697023183356317, "ece10": 0.08180750971531042, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.9377777777777778, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 1, "termination": 2, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.042167372844165135, "nll": 0.10978246418630308, "ece10": 0.09081628292237336, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.679590848235294, "nll": 0.9737320691626706, "ece10": 0.35511176470588235, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.43333333333333335, "confusion_matrix": { "false": { "false": 1, "true": 8, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.01306667755131679, "nll": 0.04767263342302212, "ece10": 0.043890482292838116, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.13019546095769463, "nll": 0.22784035537607983, "ece10": 0.13887153751798204, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8628571428571428, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 2, "thanks": 0, "fyi": 1, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.07768886085938326, "nll": 0.16374074189725868, "ece10": 0.0842379956704355, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.37694097555555556, "nll": 0.5144900216689449, "ece10": 0.2564222222222222, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6989966555183946, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 5, "accuracy": 0.3125, "wilson95": [ 0.14164643854782039, 0.5559564416525933 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3125, "brier": 0.7318873146111706, "nll": 1.1232069943153757, "ece10": 0.430280944969497, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.2863636363636364, "confusion_matrix": { "0": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.33661872643140045, "nll": 0.4984731071326667, "ece10": 0.18694284150637283, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7037037037037037, "confusion_matrix": { "0": { "0": 1, "1": 4, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 14, "accuracy": 0.7, "wilson95": [ 0.4810271816464765, 0.8545227551323956 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.4309624092553862, "nll": 0.8603899342614649, "ece10": 0.14980704281132046, "calibration_n": 20, "confident_wrong": 1, "macro_f1": 0.7011904761904761, "confusion_matrix": { "cheating": { "cheating": 3, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 2, "boosting": 0, "smurfing": 0, "other": 2, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 1, "other": 2, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 5, "accuracy": 0.23809523809523808, "wilson95": [ 0.10628013022636468, 0.4509117368295075 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.23809523809523808, "brier": 0.998475339765218, "nll": 2.2627608760137057, "ece10": 0.32101307888855934, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.1261904761904762, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 0, "punctuation": 0, "none": 2, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 0, "punctuation": 0, "none": 3, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 0, "punctuation": 0, "none": 3, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.6589999300000001, "nll": 0.9391293018610936, "ece10": 0.38345000000000007, "calibration_n": 16, "confident_wrong": 2, "macro_f1": 0.5151515151515151, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 7, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04495879551675008, "nll": 0.12220156098118512, "ece10": 0.10376202636792461, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 8, "accuracy": 0.5714285714285714, "wilson95": [ 0.3259064457965074, 0.7861920109552588 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5714285714285714, "brier": 0.5304260361490573, "nll": 1.0276702286515131, "ece10": 0.3050730814692308, "calibration_n": 14, "confident_wrong": 1, "macro_f1": 0.4666666666666666, "confusion_matrix": { "0": { "0": 1, "1": 0, "2": 2, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 2, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.5214874318882267, "nll": 0.8276291121596404, "ece10": 0.36462680415983056, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.461038961038961, "confusion_matrix": { "0": { "0": 2, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 0, "3": 2, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.5682015666649072, "nll": 1.116310474449773, "ece10": 0.16465838490095888, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6833333333333333, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 1, "3": 1, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 2, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.5016490705882353, "nll": 0.6888135200939958, "ece10": 0.25877647058823533, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.32, "confusion_matrix": { "false": { "false": 0, "true": 8, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.09176598279621122, "nll": 0.20012540926831457, "ece10": 0.11962010173973042, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9377777777777778, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "sports": 2, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.07121610872155036, "nll": 0.1612969182151467, "ece10": 0.13265831937617142, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.3485230800000001, "nll": 0.5323705383094856, "ece10": 0.08694117647058824, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.7571428571428571, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.38562591625, "nll": 0.5762819113899705, "ece10": 0.23734375000000002, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8117647058823529, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 2, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.3683517898930849, "nll": 0.7699182076998492, "ece10": 0.20951763435205134, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8193732193732194, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.25123284691418774, "nll": 0.5550492826876456, "ece10": 0.2533672168401894, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8412698412698414, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 1, "french": 3, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 1, "indian": 0, "thai": 0, "middle_eastern": 2, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.6705754070588235, "nll": 0.9763240626727567, "ece10": 0.28602352941176473, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.3952569169960475, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.43575347076876575, "nll": 0.751825538071928, "ece10": 0.2325782160234488, "calibration_n": 18, "confident_wrong": 2, "macro_f1": 0.7857142857142857, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 2, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 0, "changed_mind": 3, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.4702045459175797, "nll": 1.092117233006255, "ece10": 0.27350059216767253, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8357142857142857, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 6, "accuracy": 0.375, "wilson95": [ 0.18481232558863633, 0.6135895945449727 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.375, "brier": 0.6008569200000001, "nll": 0.7951094000305655, "ece10": 0.317, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.36507936507936506, "confusion_matrix": { "false": { "false": 4, "true": 6, "__failed__": 0 }, "true": { "false": 4, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 10, "accuracy": 0.6666666666666666, "wilson95": [ 0.41713547738519774, 0.8482367556028526 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.46286133733333334, "nll": 0.669235594476158, "ece10": 0.08319333333333334, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.6666666666666666, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 2, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.19817459660049191, "nll": 0.3560741680781455, "ece10": 0.1369301187135365, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.8580086580086579, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 1, "tech": 0, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 1, "sales": 0, "account": 0, "other": 1, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.09784077882352941, "nll": 0.1789368819233081, "ece10": 0.14144705882352943, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8819444444444444, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 11, "accuracy": 0.5789473684210527, "wilson95": [ 0.3627590353425504, 0.7685810871754576 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5789473684210527, "brier": 0.5434316549252151, "nll": 0.8866786034115892, "ece10": 0.13939573139806258, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.47077922077922074, "confusion_matrix": { "0": { "0": 3, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.5050745635294117, "nll": 0.7072441790900761, "ece10": 0.20713529411764708, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5096153846153846, "confusion_matrix": { "false": { "false": 10, "true": 1, "__failed__": 0 }, "true": { "false": 5, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.10106845555555556, "nll": 0.177279856731134, "ece10": 0.13521111111111112, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8831168831168831, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 10, "accuracy": 0.5, "wilson95": [ 0.2992980081982123, 0.7007019918017877 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.5953232933333602, "nll": 0.9493823652686848, "ece10": 0.13149520451460206, "calibration_n": 20, "confident_wrong": 1, "macro_f1": 0.5042207792207791, "confusion_matrix": { "0": { "0": 2, "1": 4, "2": 1, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 3, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.04327535249757715, "nll": 0.07942017297510932, "ece10": 0.05898374014158737, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 12, "accuracy": 0.6, "wilson95": [ 0.3865815007622531, 0.781193467627183 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5260116280000001, "nll": 0.7448860370118017, "ece10": 0.19560000000000002, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.4666666666666667, "confusion_matrix": { "false": { "false": 1, "true": 8, "__failed__": 0 }, "true": { "false": 0, "true": 11, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.316270870272793, "nll": 0.589627694087329, "ece10": 0.27345425548517893, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8647727272727272, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 2, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 431, "accuracy": 0.47888888888888886, "wilson95": [ 0.44641098408388713, 0.5115462443336039 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 561, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5228331780055918, "cluster_bootstrap95": [ 0.45102548056820624, 0.5059847660500544 ], "independent_source_groups": 596, "brier": 0.681362237679447, "nll": 1.308164523841702, "ece10": 0.2869459399549908, "calibration_n": 1073, "confident_wrong": 115 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 320, "accuracy": 0.5574912891986062, "wilson95": [ 0.5166102079293301, 0.5976079724945843 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 652, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6965811965811965, "brier": 0.45902673129556887, "nll": 1.0139529263556917, "ece10": 0.14660244114931448, "calibration_n": 936, "confident_wrong": 80 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 178, "accuracy": 0.2542857142857143, "wilson95": [ 0.22342833994256509, 0.2878252304617963 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 412, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3786764705882353, "cluster_bootstrap95": [ 0.2226950354609929, 0.28716145356990425 ], "independent_source_groups": 650, "brier": 0.8033754576613327, "nll": 1.6336500226910273, "ece10": 0.21881974258446693, "calibration_n": 1088, "confident_wrong": 56 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 20, "accuracy": 0.05, "wilson95": [ 0.03259742983714726, 0.07596363506371961 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1018, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.509, "brier": 0.648002474711062, "nll": 1.1822978360170546, "ece10": 0.1679181443580111, "calibration_n": 2000, "confident_wrong": 134 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "wilson95": [ 0.7735463743160095, 0.9801879358846942 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 26, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.1102953249622896, "nll": 0.18168945343094914, "ece10": 0.057674426632601396, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9302884615384617, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 1, "inspection": 0, "purchase": 0, "work": 6, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "wilson95": [ 0.7735463743160095, 0.9801879358846942 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 26, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.09996711528763076, "nll": 0.1782139696545141, "ece10": 0.10107970791929706, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9302884615384616, "confusion_matrix": { "work": { "work": 6, "purchase": 0, "inspection": 0, "design": 1, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 6, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.09497077348774217, "nll": 0.20080261836840047, "ece10": 0.0996774338244172, "calibration_n": 28, "confident_wrong": 1, "macro_f1": 0.9641025641025641, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 25, "accuracy": 0.8928571428571429, "wilson95": [ 0.7280414845132785, 0.9628816331707 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 25, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8928571428571429, "brier": 0.1222547301270044, "nll": 0.23240496431113494, "ece10": 0.102193360143255, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.8945741758241759, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 6, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 0, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 1, "purchase": 0, "inspection": 0, "design": 6, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 34, "accuracy": 0.85, "wilson95": [ 0.709276756335103, 0.9293881228267964 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.22099535780977858, "nll": 0.40931853126519374, "ece10": 0.10949268385425183, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8397332698261492, "confusion_matrix": { "review": { "review": 4, "it": 0, "documents": 1, "procurement": 1, "maintenance": 2, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 1, "it": 0, "documents": 6, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.21155756905621104, "nll": 0.40545597947535617, "ece10": 0.11701773651460236, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8649896800825594, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 7, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 1, "documents": 1, "it": 0, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 28, "accuracy": 0.7, "wilson95": [ 0.5456998118185507, 0.8192515477025347 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.38849613009510653, "nll": 0.7263525636254164, "ece10": 0.11442820497860703, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.6889682539682539, "confusion_matrix": { "review": { "review": 5, "it": 0, "documents": 0, "procurement": 1, "maintenance": 2, "__failed__": 0 }, "it": { "review": 0, "it": 7, "documents": 0, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "documents": { "review": 3, "it": 1, "documents": 3, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "procurement": { "review": 2, "it": 0, "documents": 1, "procurement": 5, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 30, "accuracy": 0.75, "wilson95": [ 0.5980603857923197, 0.858128813609037 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 30, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.36671457174796696, "nll": 0.6749676641048941, "ece10": 0.07619873817521239, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.734511077158136, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 7, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 1, "documents": 5, "it": 1, "review": 0, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 7, "review": 0, "__failed__": 0 }, "review": { "maintenance": 3, "procurement": 1, "documents": 1, "it": 0, "review": 3, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 321, "accuracy": 0.8025, "wilson95": [ 0.7606877261203903, 0.8385573358073602 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 321, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8025, "brier": 0.279686004557723, "nll": 0.5308759063241488, "ece10": 0.043295698087811, "calibration_n": 400, "confident_wrong": 4, "macro_f1": 0.7043302400836495, "confusion_matrix": { "sadness": { "sadness": 103, "joy": 8, "love": 3, "anger": 3, "fear": 2, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 4, "joy": 105, "love": 6, "anger": 1, "fear": 0, "surprise": 2, "__failed__": 0 }, "love": { "sadness": 2, "joy": 18, "love": 14, "anger": 1, "fear": 1, "surprise": 1, "__failed__": 0 }, "anger": { "sadness": 4, "joy": 3, "love": 2, "anger": 52, "fear": 2, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 3, "joy": 0, "love": 0, "anger": 5, "fear": 43, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 1, "love": 1, "anger": 1, "fear": 3, "surprise": 4, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 275, "accuracy": 0.9166666666666666, "wilson95": [ 0.8798783980359391, 0.9429191259436057 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 275, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.11696324588750315, "nll": 0.2877465171637354, "ece10": 0.03672908070516201, "calibration_n": 300, "confident_wrong": 2 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 367, "accuracy": 0.9175, "wilson95": [ 0.8864031104605777, 0.9406541237702847 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 367, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9175, "brier": 0.12765118139999987, "nll": 0.22965262908443773, "ece10": 0.03100899999999999, "calibration_n": 400, "confident_wrong": 9, "macro_f1": 0.8666545443525169, "confusion_matrix": { "false": { "false": 307, "true": 2, "__failed__": 0 }, "true": { "false": 31, "true": 60, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 362, "accuracy": 0.905, "wilson95": [ 0.8722916530824945, 0.9300033885067254 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 362, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.905, "brier": 0.1616037121866619, "nll": 0.3305750489316551, "ece10": 0.02877398446661736, "calibration_n": 400, "confident_wrong": 11, "macro_f1": 0.897061245915629, "confusion_matrix": { "world": { "world": 89, "sports": 6, "business": 6, "sci_tech": 2, "__failed__": 0 }, "sports": { "world": 2, "sports": 120, "business": 0, "sci_tech": 1, "__failed__": 0 }, "business": { "world": 3, "sports": 0, "business": 62, "sci_tech": 7, "__failed__": 0 }, "sci_tech": { "world": 3, "sports": 0, "business": 8, "sci_tech": 91, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 370, "accuracy": 0.925, "wilson95": [ 0.8949522937386957, 0.9469622560771523 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 370, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.10506900874999997, "nll": 0.16284245679976583, "ece10": 0.024352750000000003, "calibration_n": 400, "confident_wrong": 4, "macro_f1": 0.9183695681750157, "confusion_matrix": { "true": { "true": 128, "false": 25, "__failed__": 0 }, "false": { "true": 5, "false": 242, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 205, "accuracy": 0.6833333333333333, "wilson95": [ 0.628659261236736, 0.7333716493142638 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 205, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6833333333333333, "brier": 0.4850085617355877, "nll": 1.3827351143743383, "ece10": 0.08031329169741912, "calibration_n": 300, "confident_wrong": 3 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 388, "accuracy": 0.97, "wilson95": [ 0.9483014642909042, 0.9827569790348569 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 388, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.97, "brier": 0.03998658110000004, "nll": 0.07216154673911113, "ece10": 0.0120095, "calibration_n": 400, "confident_wrong": 3, "macro_f1": 0.9699519230769231, "confusion_matrix": { "false": { "false": 186, "true": 2, "__failed__": 0 }, "true": { "false": 10, "true": 202, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 297, "accuracy": 0.7425, "wilson95": [ 0.697484812382584, 0.7829017248652822 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 297, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7425, "brier": 0.36780524050000024, "nll": 0.5933205931877826, "ece10": 0.10865149999999997, "calibration_n": 400, "confident_wrong": 26, "macro_f1": 0.7297137721329652, "confusion_matrix": { "false": { "false": 192, "true": 8, "__failed__": 0 }, "true": { "false": 95, "true": 105, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 64, "accuracy": 0.64, "wilson95": [ 0.5423540245160874, 0.7272877959859567 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 64, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.64, "brier": 0.4605931802000001, "nll": 0.654554901145877, "ece10": 0.111859, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5535714285714286, "confusion_matrix": { "yes": { "yes": 54, "no": 0, "__failed__": 0 }, "no": { "yes": 36, "no": 10, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 49, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.7405856732049834, "nll": 1.8737300104073396, "ece10": 0.2531826364436966, "calibration_n": 100, "confident_wrong": 9, "macro_f1": 0.5164176552721444, "confusion_matrix": { "balance": { "balance": 3, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 1, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 2, "loan_payment": 1, "mortgage": 3, "wire": 1, "statement": 1, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 5, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 3, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 5, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 3, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 1, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 6, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 4, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 4, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 1, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 3, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 2, "card_lost": 0, "dispute_charge": 3, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 55, "accuracy": 0.55, "wilson95": [ 0.45244602997442135, 0.6438546202048803 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 150, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3038982619454435, "nll": 0.6215600430837572, "ece10": 0.07197532219226871, "calibration_n": 200, "confident_wrong": 2 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 130, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.5460586196780465, "nll": 1.1760439437264154, "ece10": 0.17663749745483504, "calibration_n": 200, "confident_wrong": 22 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 76, "accuracy": 0.76, "wilson95": [ 0.6676766365018553, 0.8330867444305122 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 76, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.3986334667477189, "nll": 1.2130083139915981, "ece10": 0.13762780998178326, "calibration_n": 100, "confident_wrong": 9, "macro_f1": 0.7317533312098529, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 3, "receipt": 12, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 5, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "bank_statement": { "invoice": 3, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 11, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 1, "bank_statement": 0, "clinical_note": 6, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 9, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 2, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 4, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 4, "accuracy": 0.04, "wilson95": [ 0.01566330399154762, 0.0983707143588792 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 203, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5075, "brier": 0.7575307335753796, "nll": 1.9885706161078844, "ece10": 0.28958014418260225, "calibration_n": 400, "confident_wrong": 60 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.5591102587135821, "nll": 1.2572912774428529, "ece10": 0.12371379164585539, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.5686863998070895, "confusion_matrix": { "politics": { "politics": 3, "business": 0, "technology": 2, "science": 4, "health": 3, "sports": 0, "entertainment": 1, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "business": { "politics": 1, "business": 8, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 1, "technology": 9, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 1, "technology": 0, "science": 8, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 1, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 5, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 2, "business": 2, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "world": { "politics": 1, "business": 6, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 1, "sports": 0, "entertainment": 0, "world": 1, "culture": 6, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 2, "technology": 0, "science": 2, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 4, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 39, "accuracy": 0.39, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.39, "brier": 0.8215339296884904, "nll": 2.0821746454704924, "ece10": 0.17472185524652845, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.35705882352941176, "confusion_matrix": { "biology": { "biology": 4, "medicine": 0, "chemistry": 2, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 2, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 6, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 2, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 4, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 1, "medicine": 1, "chemistry": 1, "physics": 7, "computer_science": 0, "mathematics": 0, "earth_science": 6, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 1, "chemistry": 2, "physics": 0, "computer_science": 3, "mathematics": 0, "earth_science": 5, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 1, "chemistry": 1, "physics": 2, "computer_science": 0, "mathematics": 1, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 0, "earth_science": 8, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 0, "engineering": 5, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 4, "engineering": 0, "economics": 1, "other": 0, "__failed__": 0 }, "other": { "biology": 1, "medicine": 0, "chemistry": 0, "physics": 2, "computer_science": 1, "mathematics": 0, "earth_science": 4, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 53, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.265, "brier": 0.9676345026422339, "nll": 2.7226018346966208, "ece10": 0.4721433104664531, "calibration_n": 100, "confident_wrong": 33 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 7, "accuracy": 0.07, "wilson95": [ 0.03431926106727268, 0.137495147390735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 85, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.425, "brier": 0.3472227483803529, "nll": 0.5966926601600122, "ece10": 0.09190007602883077, "calibration_n": 100, "confident_wrong": 6 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 77, "accuracy": 0.77, "wilson95": [ 0.6784561697712622, 0.8415673411969654 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 77, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.77, "brier": 0.3077740554687175, "nll": 0.5193848243836595, "ece10": 0.051505992031459925, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.7644464886727848, "confusion_matrix": { "negative": { "negative": 35, "neutral": 1, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 15, "neutral": 17, "positive": 1, "__failed__": 0 }, "positive": { "negative": 3, "neutral": 3, "positive": 25, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 16, "accuracy": 0.16, "wilson95": [ 0.10095288488804782, 0.24420269389270244 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 95, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.475, "brier": 0.32009660540000007, "nll": 0.4612041274083477, "ece10": 0.13826700000000003, "calibration_n": 100, "confident_wrong": 5 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 40, "accuracy": 0.4, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 222, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.74, "brier": 0.3494893276710086, "nll": 0.7906161355430668, "ece10": 0.07200494207478342, "calibration_n": 300, "confident_wrong": 6 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 54, "accuracy": 0.54, "wilson95": [ 0.44264860323368227, 0.6343919169097589 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 54, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.54, "brier": 0.6266533207279164, "nll": 2.215722202516952, "ece10": 0.18191439186219205, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.4737914862914862, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 2, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 1, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 4, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 2, "downgrade_plan": 2, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 1, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 2, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 3, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 1, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 4, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 34, "accuracy": 0.34, "wilson95": [ 0.25461520797348164, 0.43722271145275377 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.34, "brier": 1.0481777700455503, "nll": 2.8809668340420784, "ece10": 0.41119007259344864, "calibration_n": 100, "confident_wrong": 16, "macro_f1": 0.3287925355327316, "confusion_matrix": { "billing": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 4, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 1, "account_access": 0, "shipping": 5, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 0, "shipping": 1, "returns": 0, "technical_outage": 1, "onboarding": 0, "integrations": 4, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 3, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 2, "integrations": 3, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 5, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 1, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 1, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 3, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 4, "account_access": 0, "shipping": 3, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 9, "accuracy": 0.09, "wilson95": [ 0.04807254000256516, 0.1622621285271631 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 117, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.39, "brier": 0.8049867211717638, "nll": 1.7280174227925902, "ece10": 0.2721420584366434, "calibration_n": 300, "confident_wrong": 32 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 51, "accuracy": 0.51, "wilson95": [ 0.4134801193402717, 0.6057800106955886 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 51, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.51, "brier": 0.6955717756160422, "nll": 1.6369399420490711, "ece10": 0.23952902119769212, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.46883009492908295, "confusion_matrix": { "new_booking": { "new_booking": 2, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 1, "change_flight": 7, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 1, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 4, "baggage": 0, "refund": 2, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 2, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 8, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 2, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 4, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 3, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 3, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 1, "change_flight": 2, "cancel_trip": 1, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "other": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 1, "hotel": 4, "car": 1, "visa_docs": 0, "loyalty": 2, "disruption": 1, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 94, "accuracy": 0.94, "wilson95": [ 0.8752318455410406, 0.9722138760368118 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 94, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.94, "brier": 0.09828238203784662, "nll": 0.2109945961057407, "ece10": 0.028136336464413353, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.9383566135484295, "confusion_matrix": { "business": { "business": 24, "science and technology": 1, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 3, "science and technology": 20, "sports": 1, "world": 1, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 82, "accuracy": 0.82, "wilson95": [ 0.7333264161369926, 0.8829977450105364 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 82, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.82, "brier": 0.2473982081377115, "nll": 0.6354698674005121, "ece10": 0.0553592609216771, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.7810185185185184, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 2, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 2, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 2, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 2, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 1, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 1, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 69, "accuracy": 0.69, "wilson95": [ 0.5937394428051057, 0.7722030278762395 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.69, "brier": 0.41159619244672396, "nll": 0.8102567621050895, "ece10": 0.08782039220401852, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.6844514427847761, "confusion_matrix": { "anger": { "anger": 15, "fear": 2, "joy": 0, "love": 0, "sadness": 0, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 0, "fear": 14, "joy": 0, "love": 0, "sadness": 3, "surprise": 0, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 14, "love": 2, "sadness": 1, "surprise": 0, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 6, "love": 8, "sadness": 2, "surprise": 1, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 2, "joy": 3, "love": 0, "sadness": 11, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 2, "joy": 4, "love": 0, "sadness": 2, "surprise": 7, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2423, "accuracy": 0.8219131614654003, "wilson95": [ 0.8076862420608573, 0.835302220049259 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2423, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8219131614654003, "brier": 0.2721140506390297, "nll": 0.6403450307960039, "ece10": 0.0337925526410853, "calibration_n": 2948, "confident_wrong": 85, "macro_f1": 0.8168136556678262, "confusion_matrix": { "alarm": { "alarm": 76, "audio": 0, "calendar": 8, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 11, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 46, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 4, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 0, "audio": 0, "calendar": 376, "cooking": 0, "datetime": 3, "email": 0, "general": 6, "iot": 2, "lists": 1, "music": 0, "news": 3, "play": 0, "qa": 1, "recommendation": 5, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 69, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 75, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 22, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 0, "email": 230, "general": 22, "iot": 0, "lists": 3, "music": 0, "news": 1, "play": 0, "qa": 4, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 12, "cooking": 9, "datetime": 0, "email": 2, "general": 79, "iot": 8, "lists": 0, "music": 2, "news": 6, "play": 17, "qa": 28, "recommendation": 12, "social": 1, "takeaway": 0, "transport": 6, "weather": 5, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 8, "datetime": 0, "email": 0, "general": 0, "iot": 188, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 13, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 7, "cooking": 2, "datetime": 0, "email": 0, "general": 7, "iot": 6, "lists": 114, "music": 4, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 73, "news": 0, "play": 2, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 0, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 110, "play": 0, "qa": 9, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 4, "calendar": 1, "cooking": 0, "datetime": 0, "email": 1, "general": 27, "iot": 9, "lists": 1, "music": 61, "news": 0, "play": 277, "qa": 0, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 1, "general": 21, "iot": 2, "lists": 0, "music": 1, "news": 5, "play": 1, "qa": 247, "recommendation": 5, "social": 0, "takeaway": 0, "transport": 2, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 9, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 6, "play": 1, "qa": 3, "recommendation": 69, "social": 0, "takeaway": 2, "transport": 4, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 2, "general": 16, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 84, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 3, "social": 0, "takeaway": 50, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 120, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 140, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2021, "accuracy": 0.6855495251017639, "wilson95": [ 0.6685569969442564, 0.7020591134606262 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2021, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6855495251017639, "brier": 0.43090724478847353, "nll": 1.048759379560718, "ece10": 0.0728103468041171, "calibration_n": 2948, "confident_wrong": 87, "macro_f1": 0.6752555797474659, "confusion_matrix": { "alarm": { "alarm": 78, "audio": 0, "calendar": 6, "cooking": 0, "datetime": 3, "email": 0, "general": 1, "iot": 6, "lists": 0, "music": 0, "news": 0, "play": 2, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 32, "calendar": 1, "cooking": 3, "datetime": 2, "email": 0, "general": 11, "iot": 5, "lists": 0, "music": 2, "news": 0, "play": 1, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 2, "audio": 0, "calendar": 334, "cooking": 4, "datetime": 7, "email": 2, "general": 11, "iot": 12, "lists": 1, "music": 0, "news": 12, "play": 0, "qa": 3, "recommendation": 6, "social": 1, "takeaway": 0, "transport": 2, "weather": 3, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 67, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 9, "cooking": 0, "datetime": 71, "email": 1, "general": 0, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 11, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 2, "weather": 2, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 1, "email": 227, "general": 24, "iot": 2, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 3, "recommendation": 3, "social": 1, "takeaway": 0, "transport": 2, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 2, "calendar": 13, "cooking": 14, "datetime": 2, "email": 2, "general": 70, "iot": 16, "lists": 1, "music": 2, "news": 8, "play": 7, "qa": 23, "recommendation": 13, "social": 0, "takeaway": 1, "transport": 7, "weather": 6, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 1, "calendar": 0, "cooking": 10, "datetime": 0, "email": 0, "general": 6, "iot": 174, "lists": 0, "music": 2, "news": 0, "play": 1, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 17, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 3, "datetime": 0, "email": 0, "general": 7, "iot": 4, "lists": 106, "music": 5, "news": 1, "play": 5, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 3, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 0, "email": 0, "general": 2, "iot": 1, "lists": 0, "music": 67, "news": 2, "play": 3, "qa": 2, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 2, "datetime": 0, "email": 1, "general": 6, "iot": 0, "lists": 1, "music": 0, "news": 95, "play": 2, "qa": 8, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 2, "weather": 1, "__failed__": 0 }, "play": { "alarm": 0, "audio": 58, "calendar": 3, "cooking": 10, "datetime": 3, "email": 1, "general": 33, "iot": 13, "lists": 7, "music": 93, "news": 4, "play": 150, "qa": 1, "recommendation": 5, "social": 0, "takeaway": 2, "transport": 2, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 10, "datetime": 0, "email": 0, "general": 25, "iot": 19, "lists": 0, "music": 4, "news": 9, "play": 3, "qa": 190, "recommendation": 9, "social": 0, "takeaway": 0, "transport": 9, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 11, "cooking": 0, "datetime": 1, "email": 0, "general": 2, "iot": 3, "lists": 0, "music": 1, "news": 5, "play": 6, "qa": 2, "recommendation": 56, "social": 0, "takeaway": 1, "transport": 6, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 2, "general": 18, "iot": 5, "lists": 0, "music": 0, "news": 4, "play": 0, "qa": 0, "recommendation": 1, "social": 75, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 6, "datetime": 0, "email": 0, "general": 4, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 42, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 1, "datetime": 2, "email": 0, "general": 2, "iot": 3, "lists": 0, "music": 1, "news": 1, "play": 0, "qa": 1, "recommendation": 6, "social": 0, "takeaway": 5, "transport": 99, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 1, "audio": 0, "calendar": 7, "cooking": 1, "datetime": 2, "email": 1, "general": 14, "iot": 11, "lists": 0, "music": 0, "news": 8, "play": 1, "qa": 4, "recommendation": 5, "social": 0, "takeaway": 0, "transport": 9, "weather": 88, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 793, "accuracy": 0.6760443307757886, "wilson95": [ 0.6487260908882926, 0.7022132789855942 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 793, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6760443307757886, "cluster_bootstrap95": [ 0.6429700892626877, 0.7083363702623907 ], "independent_source_groups": 47, "brier": 0.4485972440590523, "nll": 0.8197470885467726, "ece10": 0.0621298706239168, "calibration_n": 1173, "confident_wrong": 21, "macro_f1": 0.625183853209179, "confusion_matrix": { "negative": { "negative": 87, "neutral": 81, "positive": 14, "__failed__": 0 }, "neutral": { "negative": 58, "neutral": 484, "positive": 48, "__failed__": 0 }, "positive": { "negative": 33, "neutral": 146, "positive": 222, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "decider-2b": { "label": "Decider 2B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 176, "accuracy": 0.88, "wilson95": [ 0.8276574790170326, 0.9180200729362449 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 176, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.17223786810478994, "nll": 0.31843526260838534, "ece10": 0.041124973905329716, "calibration_n": 200, "confident_wrong": 2, "macro_f1": 0.8818313674961529, "confusion_matrix": { "world": { "world": 46, "sports": 0, "business": 0, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 5, "sports": 0, "business": 38, "sci_tech": 8, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 4, "sci_tech": 46, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 166, "accuracy": 0.83, "wilson95": [ 0.7718411637168737, 0.8757209208741302 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 166, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.83, "brier": 0.23733177008921427, "nll": 0.43253178873979325, "ece10": 0.036264731075855476, "calibration_n": 200, "confident_wrong": 6, "macro_f1": 0.7000987613529666, "confusion_matrix": { "sadness": { "sadness": 52, "joy": 0, "love": 1, "anger": 4, "fear": 0, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 4, "joy": 69, "love": 1, "anger": 2, "fear": 1, "surprise": 2, "__failed__": 0 }, "love": { "sadness": 1, "joy": 8, "love": 3, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 1, "joy": 2, "love": 1, "anger": 22, "fear": 1, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 1, "joy": 1, "love": 0, "anger": 0, "fear": 18, "surprise": 0, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 1, "love": 0, "anger": 0, "fear": 1, "surprise": 2, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 196, "accuracy": 0.98, "wilson95": [ 0.9497129130941735, 0.9921955735836505 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 196, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.98, "brier": 0.05930908419999997, "nll": 0.135292624277925, "ece10": 0.086865, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.9797550359348112, "confusion_matrix": { "false": { "false": 109, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 87, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": 0.8, "development": { "n": 86, "correct": 86, "wilson95": [ 0.9572418027142576, 0.9999999999999999 ] }, "test_coverage": 0.83, "accepted_n": 166, "accepted_correct": 166, "accepted_accuracy": 1.0, "accepted_wilson95": [ 0.9773820900540565, 0.9999999999999999 ], "errors_per_1000_all_test_cases": 0.0, "note": "Selected on development only, applied unchanged to test. An empirical pilot estimate, not a production guarantee." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 189, "accuracy": 0.945, "wilson95": [ 0.9042130012276914, 0.9690146582965411 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 189, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.945, "brier": 0.09220854219999998, "nll": 0.17459096294086515, "ece10": 0.05476299999999999, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.9423344080102749, "confusion_matrix": { "true": { "true": 73, "false": 4, "__failed__": 0 }, "false": { "true": 7, "false": 116, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": 0.8, "development": { "n": 83, "correct": 83, "wilson95": [ 0.9557646903580262, 1.0 ] }, "test_coverage": 0.865, "accepted_n": 173, "accepted_correct": 170, "accepted_accuracy": 0.9826589595375722, "accepted_wilson95": [ 0.9502634429465275, 0.9940852471947772 ], "errors_per_1000_all_test_cases": 15.0, "note": "Selected on development only, applied unchanged to test. An empirical pilot estimate, not a production guarantee." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3751255240910725, "nll": 0.6541998788091811, "ece10": 0.19004274934784415, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.7733333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 6, "accuracy": 0.5, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.5419372916433942, "nll": 1.002585033085724, "ece10": 0.20178387411258875, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.4766666666666667, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 1, "review": 1, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.021260212664290967, "nll": 0.09020475083382971, "ece10": 0.08365965881934657, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.10010040642483241, "nll": 0.2024354072844303, "ece10": 0.1616892152233088, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9142857142857144, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 2, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 93, "accuracy": 0.93, "wilson95": [ 0.8625048526092651, 0.9656807389327273 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 93, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.93, "brier": 0.11677029770989236, "nll": 0.2583018179228779, "ece10": 0.0662594706914347, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.9273629964806436, "confusion_matrix": { "label_000": { "label_000": 24, "label_001": 1, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 3, "label_001": 19, "label_002": 1, "label_003": 2, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 76, "accuracy": 0.76, "wilson95": [ 0.6676766365018553, 0.8330867444305122 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 76, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.37352661513949004, "nll": 0.7292537890853679, "ece10": 0.0750007590685273, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.7576637243303909, "confusion_matrix": { "label_000": { "label_000": 14, "label_001": 3, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 15, "label_002": 0, "label_003": 0, "label_004": 1, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 14, "label_003": 1, "label_004": 2, "label_005": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 6, "label_003": 9, "label_004": 2, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 1, "label_001": 0, "label_002": 1, "label_003": 0, "label_004": 14, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 1, "label_002": 2, "label_003": 0, "label_004": 2, "label_005": 10, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 78, "accuracy": 0.78, "wilson95": [ 0.6892964648501421, 0.8499871761539459 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 78, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.78, "brier": 0.2732197695263257, "nll": 0.5978006278578583, "ece10": 0.047288752410928815, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.7352513227513228, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 2, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 1, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 2, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 1, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 1, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 1, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.001306716050661562, "nll": 0.016097891934820304, "ece10": 0.01569131588053814, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 63, "accuracy": 0.5675675675675675, "wilson95": [ 0.47467121788751543, 0.6559436328295938 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 63, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5675675675675675, "brier": 0.5566137527611468, "nll": 0.9612859501568943, "ece10": 0.1838681312349606, "calibration_n": 111, "confident_wrong": 3 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 64, "accuracy": 0.8888888888888888, "wilson95": [ 0.7957706176633595, 0.9426118036505066 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 64, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "cluster_bootstrap95": [ 0.7916666666666666, 0.9722222222222222 ], "independent_source_groups": 36, "brier": 0.17457486355607582, "nll": 0.30473934330492874, "ece10": 0.07295310750155885, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 15, "accuracy": 0.7894736842105263, "wilson95": [ 0.5666572155618583, 0.9149232336708375 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7894736842105263, "brier": 0.21148606, "nll": 0.35004924599241916, "ece10": 0.2290578947368421, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.7840909090909092, "confusion_matrix": { "false": { "false": 9, "true": 2, "__failed__": 0 }, "true": { "false": 2, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.25934932875000005, "nll": 0.383315187953878, "ece10": 0.13973124999999997, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8056680161943319, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 17, "accuracy": 0.7727272727272727, "wilson95": [ 0.5656004682477829, 0.8987696001475147 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7727272727272727, "brier": 0.26904996, "nll": 0.436764246031211, "ece10": 0.21111818181818182, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.7684210526315789, "confusion_matrix": { "false": { "false": 7, "true": 3, "__failed__": 0 }, "true": { "false": 2, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.1200426210670699, "nll": 0.18007083601418747, "ece10": 0.11504611169696113, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.886904761904762, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 3, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 1, "feature_request": 0, "praise": 0, "pricing_billing": 3, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.2967337110091477, "nll": 0.5454365583703733, "ece10": 0.24033897188472306, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7295238095238095, "confusion_matrix": { "0": { "0": 2, "1": 2, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 3, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 2, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.017993809760775456, "nll": 0.07909971362618001, "ece10": 0.07303264287227382, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.23213106257279356, "nll": 0.44436164363434255, "ece10": 0.19826905984639648, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7444444444444444, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 2, "refactor": 0, "nit": 1, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 1, "security": 0, "style": 1, "refactor": 1, "nit": 1, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.222370759247329, "nll": 0.4472334284148811, "ece10": 0.12341676956057035, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.837037037037037, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 0, "chore": 1, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.607841434117647, "nll": 0.8433923492487986, "ece10": 0.3743588235294118, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.43333333333333335, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 8, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.22755318, "nll": 0.3857977528668414, "ece10": 0.141025, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 11, "accuracy": 0.6111111111111112, "wilson95": [ 0.3861904160223791, 0.7969475342783638 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6111111111111112, "brier": 0.41873305136770234, "nll": 0.7788581630005866, "ece10": 0.24340881170450868, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.4390756302521009, "confusion_matrix": { "harassment": { "harassment": 3, "threat": 0, "self_harm": 0, "spam": 0, "none": 2, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 2, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 3, "none": 1, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 5, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.16714083931697576, "nll": 0.3383269436988568, "ece10": 0.13999813694316318, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 2, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 2, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03684996892577041, "nll": 0.08553639455711037, "ece10": 0.07278328873747994, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.03195212230194605, "nll": 0.07407655470088709, "ece10": 0.058637883801747254, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 1, "failed_attempt": 0, "damaged": 3, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.23338454117647056, "nll": 0.39352610439434527, "ece10": 0.17218823529411764, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8131868131868132, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.003972266800710237, "nll": 0.03566717524872519, "ece10": 0.03449836626548844, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.1717728543071179, "nll": 0.42721292716041187, "ece10": 0.09350629728372961, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8628571428571428, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 2, "thanks": 0, "fyi": 1, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.11038526348283303, "nll": 0.19770579192730506, "ece10": 0.10499641437056781, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8333333333333333, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 1, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.36948089999999995, "nll": 0.5530930558908713, "ece10": 0.13567777777777779, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.6989966555183946, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.5034783564017815, "nll": 0.7789037558944043, "ece10": 0.19878121311693714, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.625, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.2869742544522096, "nll": 0.48415298621825936, "ece10": 0.1353166696909445, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7037037037037037, "confusion_matrix": { "0": { "0": 1, "1": 4, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.24858969818254142, "nll": 0.5352596141968513, "ece10": 0.26692747759732477, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 4, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 3, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 8, "accuracy": 0.38095238095238093, "wilson95": [ 0.20751011368564493, 0.5912134622488425 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.38095238095238093, "brier": 0.8420732620647513, "nll": 1.7207030964040986, "ece10": 0.33984577226321966, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.2909090909090909, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 3, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 1, "none": 3, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.10759590875, "nll": 0.24535695384907955, "ece10": 0.21155625, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02884290632272071, "nll": 0.12774645141750685, "ece10": 0.11617685253907287, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 12, "accuracy": 0.8571428571428571, "wilson95": [ 0.6005862050302965, 0.9599060787285351 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.42001156950886276, "nll": 0.7916750661690617, "ece10": 0.3103170441071704, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.75, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3928193164075605, "nll": 0.6920272464207715, "ece10": 0.20409880363036306, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.711111111111111, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 1, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.4039034889879735, "nll": 0.7241051318540425, "ece10": 0.25612847274053474, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7048160173160173, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 2, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.37503151999999995, "nll": 0.5599359231785668, "ece10": 0.15582352941176467, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7571428571428571, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04474594444047871, "nll": 0.132908893734503, "ece10": 0.11523542251070716, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.08266456586059562, "nll": 0.19262631040881179, "ece10": 0.14080068292347742, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "payments": { "payments": 3, "web": 0, "infra": 0, "data": 1, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.27698531882352945, "nll": 0.4278689241448891, "ece10": 0.10492352941176472, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7571428571428571, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.17948721374999999, "nll": 0.34540401875051635, "ece10": 0.28648125, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.24261011201936777, "nll": 0.4702107874373001, "ece10": 0.17063825333294502, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8771043771043772, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.09521201143786223, "nll": 0.24763823674925536, "ece10": 0.13673453416484535, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571427, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 1, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.4040027011764706, "nll": 0.5928652003615288, "ece10": 0.1577411764705882, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7424242424242424, "confusion_matrix": { "false": { "false": 4, "true": 4, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.1597882028232979, "nll": 0.2853685411578868, "ece10": 0.06466032343004766, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8833333333333332, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 2, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.4488988331271692, "nll": 1.009254923951319, "ece10": 0.16572158071143805, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7753968253968254, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.40539249875, "nll": 0.582424887475918, "ece10": 0.15624375000000001, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.746031746031746, "confusion_matrix": { "false": { "false": 7, "true": 3, "__failed__": 0 }, "true": { "false": 1, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 8, "accuracy": 0.5333333333333333, "wilson95": [ 0.30116980025498397, 0.7519046463426261 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5333333333333333, "brier": 0.6179154426666665, "nll": 0.8540206284781995, "ece10": 0.2975333333333333, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.34782608695652173, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 7, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0665203136447744, "nll": 0.17480925004349795, "ece10": 0.1490523679066413, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 4, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.27093585882352944, "nll": 0.4349680594823038, "ece10": 0.11198823529411765, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8210526315789474, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 13, "accuracy": 0.6842105263157895, "wilson95": [ 0.46010407675327475, 0.8463562091220773 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6842105263157895, "brier": 0.42755881281144564, "nll": 0.7613721147077882, "ece10": 0.17812371684650033, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.5595238095238095, "confusion_matrix": { "0": { "0": 4, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.4761017576470587, "nll": 0.6703806552558896, "ece10": 0.1463529411764706, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.39285714285714285, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0731125988888889, "nll": 0.16344509310588465, "ece10": 0.1389388888888889, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 16, "accuracy": 0.8, "wilson95": [ 0.5839825677481064, 0.919342337420202 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.4356891461791651, "nll": 0.7499095194793088, "ece10": 0.29903980857539814, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8034188034188033, "confusion_matrix": { "0": { "0": 5, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.05460865787215583, "nll": 0.12553402722815923, "ece10": 0.08350448425944482, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 1, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 3, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 9, "accuracy": 0.45, "wilson95": [ 0.2581978582556243, 0.6579146575496577 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.6277156540000001, "nll": 0.8431446581952979, "ece10": 0.38653000000000004, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.3103448275862069, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 11, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.39857290652699545, "nll": 0.7471769085292282, "ece10": 0.3959134420611345, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9080419580419581, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 434, "accuracy": 0.4822222222222222, "wilson95": [ 0.44972172882539985, 0.5148738319472241 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 568, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5293569431500466, "cluster_bootstrap95": [ 0.4503348643178603, 0.5115815960341172 ], "independent_source_groups": 596, "brier": 0.6064886468429155, "nll": 1.0904405561754573, "ece10": 0.20075672497373206, "calibration_n": 1073, "confident_wrong": 49 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 373, "accuracy": 0.6498257839721254, "wilson95": [ 0.6099228478752408, 0.6877366526234454 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 723, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7724358974358975, "brier": 0.33882776382005614, "nll": 0.6629468866900385, "ece10": 0.06546479866833989, "calibration_n": 936, "confident_wrong": 42 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 269, "accuracy": 0.3842857142857143, "wilson95": [ 0.348975908853968, 0.4208586213945976 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 560, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5147058823529411, "cluster_bootstrap95": [ 0.3502772171291005, 0.4223776223776224 ], "independent_source_groups": 650, "brier": 0.6134812880533835, "nll": 1.171604312251265, "ece10": 0.12044251895760306, "calibration_n": 1088, "confident_wrong": 16 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 46, "accuracy": 0.115, "wilson95": [ 0.0873329226216324, 0.1499915440157759 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1227, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6135, "brier": 0.5255278840572717, "nll": 0.911930133293707, "ece10": 0.06765369313426348, "calibration_n": 2000, "confident_wrong": 18 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.007023654366892339, "nll": 0.04339752233464204, "ece10": 0.041246693269966925, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.011605601908053138, "nll": 0.057522521647158306, "ece10": 0.053799600602603144, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.05542431024005984, "nll": 0.11347824265428962, "ece10": 0.08553231193618024, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.04456983056648599, "nll": 0.11515223757470625, "ece10": 0.07139476166230477, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 6, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 36, "accuracy": 0.9, "wilson95": [ 0.7694822477247771, 0.9604204713173936 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 36, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.16819901898297837, "nll": 0.35268541044591056, "ece10": 0.11431144645753945, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8960683760683761, "confusion_matrix": { "review": { "review": 5, "it": 0, "documents": 0, "procurement": 1, "maintenance": 2, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 7, "procurement": 1, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.14980417337556304, "nll": 0.3269299029420788, "ece10": 0.14937235494894854, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.871473555684082, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 1, "documents": 6, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 1, "documents": 0, "it": 0, "review": 5, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 30, "accuracy": 0.75, "wilson95": [ 0.5980603857923197, 0.858128813609037 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 30, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.35479112950649877, "nll": 0.699528417809249, "ece10": 0.10252109884596097, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.746078431372549, "confusion_matrix": { "review": { "review": 4, "it": 1, "documents": 1, "procurement": 1, "maintenance": 1, "__failed__": 0 }, "it": { "review": 1, "it": 6, "documents": 0, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 7, "procurement": 1, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 1, "it": 0, "documents": 1, "procurement": 6, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 1, "it": 0, "documents": 0, "procurement": 0, "maintenance": 7, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 30, "accuracy": 0.75, "wilson95": [ 0.5980603857923197, 0.858128813609037 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 30, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.32076680977789024, "nll": 0.639897090800578, "ece10": 0.16068825419243252, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7469281045751635, "confusion_matrix": { "maintenance": { "maintenance": 7, "procurement": 0, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 6, "documents": 1, "it": 0, "review": 1, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 7, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 6, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 1, "documents": 1, "it": 1, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 340, "accuracy": 0.85, "wilson95": [ 0.8116863892693679, 0.8816550046966246 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 340, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.21031009222575567, "nll": 0.39813722593329554, "ece10": 0.06541543444354313, "calibration_n": 400, "confident_wrong": 2, "macro_f1": 0.7853360688030326, "confusion_matrix": { "sadness": { "sadness": 105, "joy": 7, "love": 1, "anger": 3, "fear": 3, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 2, "joy": 108, "love": 2, "anger": 3, "fear": 1, "surprise": 2, "__failed__": 0 }, "love": { "sadness": 2, "joy": 16, "love": 17, "anger": 1, "fear": 1, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 1, "joy": 0, "love": 1, "anger": 55, "fear": 6, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 0, "joy": 0, "love": 0, "anger": 2, "fear": 48, "surprise": 2, "__failed__": 0 }, "surprise": { "sadness": 1, "joy": 0, "love": 0, "anger": 0, "fear": 2, "surprise": 7, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 268, "accuracy": 0.8933333333333333, "wilson95": [ 0.8532967116826325, 0.9234241509540578 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 268, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8933333333333333, "brier": 0.15275932307942844, "nll": 0.3720522742148347, "ece10": 0.047033211263801854, "calibration_n": 300, "confident_wrong": 1 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 358, "accuracy": 0.895, "wilson95": [ 0.8611090149132419, 0.921376272562664 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 358, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.895, "brier": 0.16809759165000002, "nll": 0.30159160821998854, "ece10": 0.10341275, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.8506348020911128, "confusion_matrix": { "false": { "false": 288, "true": 21, "__failed__": 0 }, "true": { "false": 21, "true": 70, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 356, "accuracy": 0.89, "wilson95": [ 0.8555490895383555, 0.9170313208808931 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 356, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.89, "brier": 0.17254861874085264, "nll": 0.32765756018290787, "ece10": 0.0249195212440877, "calibration_n": 400, "confident_wrong": 8, "macro_f1": 0.8804359874075876, "confusion_matrix": { "world": { "world": 89, "sports": 6, "business": 6, "sci_tech": 2, "__failed__": 0 }, "sports": { "world": 1, "sports": 122, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 5, "sports": 0, "business": 63, "sci_tech": 4, "__failed__": 0 }, "sci_tech": { "world": 7, "sports": 0, "business": 13, "sci_tech": 82, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 376, "accuracy": 0.94, "wilson95": [ 0.9122771510995438, 0.9593520298862752 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 376, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.94, "brier": 0.07675700750000004, "nll": 0.15285321906673352, "ece10": 0.06631300000000001, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.9359999999999999, "confusion_matrix": { "true": { "true": 138, "false": 15, "__failed__": 0 }, "false": { "true": 9, "false": 238, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 173, "accuracy": 0.5766666666666667, "wilson95": [ 0.5201332672229635, 0.6312614771892728 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 173, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5766666666666667, "brier": 0.5412696868351936, "nll": 1.2951048079928174, "ece10": 0.0891941139879774, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 391, "accuracy": 0.9775, "wilson95": [ 0.9577973424412419, 0.988118416469505 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 391, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9775, "brier": 0.054463271799999984, "nll": 0.13075205035189202, "ece10": 0.0768855, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.9774683148177123, "confusion_matrix": { "false": { "false": 188, "true": 0, "__failed__": 0 }, "true": { "false": 9, "true": 203, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 266, "accuracy": 0.665, "wilson95": [ 0.617370024300236, 0.7094909185694462 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 266, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.665, "brier": 0.38006210525000006, "nll": 0.5479522095458276, "ece10": 0.12098574999999999, "calibration_n": 400, "confident_wrong": 5, "macro_f1": 0.6293837813917469, "confusion_matrix": { "false": { "false": 195, "true": 5, "__failed__": 0 }, "true": { "false": 129, "true": 71, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 81, "accuracy": 0.81, "wilson95": [ 0.7222115462093562, 0.8748524849023126 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 81, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.81, "brier": 0.2995291310000001, "nll": 0.48520664685970877, "ece10": 0.06714500000000001, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.8043455874781176, "confusion_matrix": { "yes": { "yes": 49, "no": 5, "__failed__": 0 }, "no": { "yes": 14, "no": 32, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 55, "accuracy": 0.55, "wilson95": [ 0.45244602997442135, 0.6438546202048803 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 55, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.6125607420498094, "nll": 1.5426934516841775, "ece10": 0.17199348864407266, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.5530288237089708, "confusion_matrix": { "balance": { "balance": 3, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 2, "dispute_charge": 2, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 5, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 }, "statement": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 5, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 6, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 6, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 7, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "direct_deposit": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 3, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 3, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 60, "accuracy": 0.6, "wilson95": [ 0.5020025867910618, 0.6905987135675411 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 157, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.785, "brier": 0.29027503082014616, "nll": 0.4943002982672574, "ece10": 0.040400697586633706, "calibration_n": 200, "confident_wrong": 3 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 45, "accuracy": 0.45, "wilson95": [ 0.3561453797951198, 0.5475539700255787 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 127, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.635, "brier": 0.5538726787954957, "nll": 1.1111858507002652, "ece10": 0.18804403985834434, "calibration_n": 200, "confident_wrong": 18 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 82, "accuracy": 0.82, "wilson95": [ 0.7333264161369926, 0.8829977450105364 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 82, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.82, "brier": 0.32408843968700246, "nll": 0.9225427526417662, "ece10": 0.11283994162557312, "calibration_n": 100, "confident_wrong": 8, "macro_f1": 0.7808522201561252, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 14, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 13, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 7, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 2, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 9, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 6, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 4, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 10, "accuracy": 0.1, "wilson95": [ 0.0552291370606751, 0.17436566150491345 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 218, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.545, "brier": 0.6870495421612582, "nll": 1.5637714854651745, "ece10": 0.22668056187243402, "calibration_n": 400, "confident_wrong": 53 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 62, "accuracy": 0.62, "wilson95": [ 0.5220975529551107, 0.7090240074752127 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 62, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.62, "brier": 0.5208140678749889, "nll": 1.1670917508501517, "ece10": 0.0868899107388887, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6079361942405421, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 2, "science": 3, "health": 3, "sports": 0, "entertainment": 0, "world": 1, "culture": 1, "weather": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 10, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 2, "technology": 7, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 1, "technology": 1, "science": 7, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 1, "health": 9, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 1, "health": 0, "sports": 7, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 0, "technology": 1, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 2, "weather": 0, "__failed__": 0 }, "world": { "politics": 2, "business": 3, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 1, "culture": 0, "weather": 1, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 7, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 2, "technology": 0, "science": 2, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 5, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 39, "accuracy": 0.39, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.39, "brier": 0.8147371578711363, "nll": 1.9077953695217138, "ece10": 0.1682715807179636, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.3789755468702837, "confusion_matrix": { "biology": { "biology": 5, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 3, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 0, "medicine": 6, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 3, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 3, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 2, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 1, "medicine": 1, "chemistry": 1, "physics": 4, "computer_science": 0, "mathematics": 0, "earth_science": 6, "engineering": 2, "economics": 1, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 1, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 4, "mathematics": 0, "earth_science": 2, "engineering": 8, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 1, "physics": 0, "computer_science": 1, "mathematics": 1, "earth_science": 2, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 9, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 2, "earth_science": 1, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 3, "engineering": 0, "economics": 3, "other": 0, "__failed__": 0 }, "other": { "biology": 2, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 1, "mathematics": 1, "earth_science": 3, "engineering": 3, "economics": 1, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 6, "accuracy": 0.06, "wilson95": [ 0.027786123963188192, 0.1247681544589592 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 66, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.33, "brier": 0.7326365166269619, "nll": 1.5683136308430783, "ece10": 0.3523967278552473, "calibration_n": 100, "confident_wrong": 25 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 11, "accuracy": 0.11, "wilson95": [ 0.06254196357064568, 0.18631296503080313 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 90, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.30682069397653017, "nll": 0.5261784425799995, "ece10": 0.04644858512522585, "calibration_n": 100, "confident_wrong": 2 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.25861992830151875, "nll": 0.4514689765039696, "ece10": 0.07096998394816983, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.7943648128815025, "confusion_matrix": { "negative": { "negative": 34, "neutral": 2, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 11, "neutral": 19, "positive": 3, "__failed__": 0 }, "positive": { "negative": 2, "neutral": 2, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 21, "accuracy": 0.21, "wilson95": [ 0.141656540619153, 0.29979968834089865 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 102, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.51, "brier": 0.23428982979999988, "nll": 0.35577155086513623, "ece10": 0.08776099999999999, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 44, "accuracy": 0.44, "wilson95": [ 0.34672026630160796, 0.5377189534832302 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 226, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7533333333333333, "brier": 0.3654758327891612, "nll": 0.7871053716805635, "ece10": 0.09018167516236661, "calibration_n": 300, "confident_wrong": 20 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.649482809029163, "nll": 1.8241512676995952, "ece10": 0.15302102516069827, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.5571448079906726, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 1, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 4, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 2, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 3, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 3, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 1, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 2, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 3, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 9, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 1, "tax_document": 3, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 2, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 1, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 38, "accuracy": 0.38, "wilson95": [ 0.29097599252478734, 0.47790244704488916 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.38, "brier": 0.9115242864080897, "nll": 2.333975489542194, "ece10": 0.35429386121790685, "calibration_n": 100, "confident_wrong": 12, "macro_f1": 0.37864916054571224, "confusion_matrix": { "billing": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 3, "account_access": 4, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 4, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 2, "shipping": 0, "returns": 0, "technical_outage": 1, "onboarding": 0, "integrations": 3, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 2, "integrations": 3, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 6, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 3, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 3, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 3, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "other": { "billing": 3, "account_access": 0, "shipping": 3, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 2, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 7, "accuracy": 0.07, "wilson95": [ 0.03431926106727268, 0.137495147390735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 119, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.39666666666666667, "brier": 0.8248986644813558, "nll": 1.762615785187091, "ece10": 0.3025024564865347, "calibration_n": 300, "confident_wrong": 29 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 60, "accuracy": 0.6, "wilson95": [ 0.5020025867910618, 0.6905987135675411 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 60, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5771418837249981, "nll": 1.3986096325499346, "ece10": 0.18177497927496503, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.5599777209499245, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 7, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 2, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 5, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 3, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 2, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 2, "loyalty": 2, "disruption": 0, "other": 1, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 8, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 1, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 1, "disruption": 5, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 1, "car": 1, "visa_docs": 0, "loyalty": 3, "disruption": 3, "other": 2, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 94, "accuracy": 0.94, "wilson95": [ 0.8752318455410406, 0.9722138760368118 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 94, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.94, "brier": 0.10616565694064198, "nll": 0.23834644273465921, "ece10": 0.06385590208646902, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.9381680765951866, "confusion_matrix": { "business": { "business": 24, "science and technology": 1, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 2, "science and technology": 20, "sports": 1, "world": 2, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 85, "accuracy": 0.85, "wilson95": [ 0.7671644040916763, 0.9069401471634337 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 85, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.19445389876961525, "nll": 0.4535014093897607, "ece10": 0.09276979088669812, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8292328042328042, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 2, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 2, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 2, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 1, "changing a pin code": 0, "compromised card": 1, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 2, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 2, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 1, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 1, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 1, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 79, "accuracy": 0.79, "wilson95": [ 0.7002003116591013, 0.858343459380847 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 79, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.3223855493911599, "nll": 0.6478499481574886, "ece10": 0.0713453080947231, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.7909317916501825, "confusion_matrix": { "anger": { "anger": 14, "fear": 3, "joy": 0, "love": 0, "sadness": 0, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 0, "fear": 15, "joy": 0, "love": 0, "sadness": 1, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 15, "love": 0, "sadness": 2, "surprise": 0, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 6, "love": 10, "sadness": 1, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 1, "fear": 1, "joy": 1, "love": 0, "sadness": 13, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 0, "fear": 1, "joy": 0, "love": 0, "sadness": 3, "surprise": 12, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2433, "accuracy": 0.8253052917232022, "wilson95": [ 0.8111776712314901, 0.8385862225130848 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2433, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8253052917232022, "brier": 0.2501130985838398, "nll": 0.5816498635620603, "ece10": 0.0461723663517122, "calibration_n": 2948, "confident_wrong": 33, "macro_f1": 0.8280294764244889, "confusion_matrix": { "alarm": { "alarm": 88, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 52, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 2, "audio": 0, "calendar": 358, "cooking": 0, "datetime": 7, "email": 0, "general": 9, "iot": 0, "lists": 1, "music": 1, "news": 9, "play": 0, "qa": 3, "recommendation": 4, "social": 1, "takeaway": 3, "transport": 1, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 67, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 94, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 4, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 230, "general": 22, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 1, "recommendation": 0, "social": 10, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 11, "cooking": 5, "datetime": 0, "email": 0, "general": 111, "iot": 2, "lists": 0, "music": 1, "news": 9, "play": 1, "qa": 19, "recommendation": 12, "social": 6, "takeaway": 0, "transport": 5, "weather": 5, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 2, "calendar": 1, "cooking": 7, "datetime": 0, "email": 0, "general": 14, "iot": 180, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 7, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 1, "datetime": 0, "email": 0, "general": 10, "iot": 2, "lists": 119, "music": 2, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 9, "iot": 0, "lists": 0, "music": 67, "news": 2, "play": 2, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 119, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 4, "calendar": 0, "cooking": 0, "datetime": 0, "email": 1, "general": 26, "iot": 2, "lists": 3, "music": 67, "news": 7, "play": 273, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 46, "iot": 0, "lists": 0, "music": 2, "news": 14, "play": 0, "qa": 212, "recommendation": 4, "social": 1, "takeaway": 0, "transport": 2, "weather": 5, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 2, "news": 16, "play": 1, "qa": 0, "recommendation": 64, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 15, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 0, "recommendation": 0, "social": 89, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 3, "social": 0, "takeaway": 49, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 1, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 115, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 3, "iot": 1, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 146, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2099, "accuracy": 0.7120081411126187, "wilson95": [ 0.6953943437102668, 0.7280701334433923 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2099, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7120081411126187, "brier": 0.3875689383644532, "nll": 0.9425637344363169, "ece10": 0.044379878013917066, "calibration_n": 2948, "confident_wrong": 17, "macro_f1": 0.7315376216603601, "confusion_matrix": { "alarm": { "alarm": 85, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 0, "general": 2, "iot": 4, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 38, "calendar": 0, "cooking": 1, "datetime": 2, "email": 0, "general": 17, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 6, "audio": 0, "calendar": 311, "cooking": 2, "datetime": 6, "email": 1, "general": 33, "iot": 1, "lists": 1, "music": 2, "news": 16, "play": 0, "qa": 3, "recommendation": 7, "social": 1, "takeaway": 4, "transport": 4, "weather": 2, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 63, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 88, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 4, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 2, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 221, "general": 33, "iot": 0, "lists": 2, "music": 0, "news": 5, "play": 0, "qa": 0, "recommendation": 0, "social": 3, "takeaway": 0, "transport": 1, "weather": 2, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 8, "cooking": 5, "datetime": 1, "email": 1, "general": 113, "iot": 2, "lists": 0, "music": 2, "news": 19, "play": 2, "qa": 10, "recommendation": 7, "social": 1, "takeaway": 1, "transport": 6, "weather": 8, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 5, "calendar": 0, "cooking": 11, "datetime": 1, "email": 0, "general": 31, "iot": 145, "lists": 0, "music": 1, "news": 0, "play": 1, "qa": 0, "recommendation": 3, "social": 0, "takeaway": 13, "transport": 0, "weather": 2, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 2, "calendar": 4, "cooking": 1, "datetime": 0, "email": 0, "general": 15, "iot": 1, "lists": 111, "music": 3, "news": 1, "play": 2, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 8, "iot": 0, "lists": 0, "music": 65, "news": 2, "play": 4, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 1, "news": 115, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 3, "__failed__": 0 }, "play": { "alarm": 0, "audio": 9, "calendar": 0, "cooking": 1, "datetime": 1, "email": 0, "general": 64, "iot": 1, "lists": 6, "music": 109, "news": 17, "play": 167, "qa": 0, "recommendation": 1, "social": 1, "takeaway": 2, "transport": 4, "weather": 2, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 1, "datetime": 0, "email": 0, "general": 82, "iot": 0, "lists": 0, "music": 1, "news": 29, "play": 1, "qa": 157, "recommendation": 5, "social": 0, "takeaway": 1, "transport": 3, "weather": 5, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 9, "cooking": 0, "datetime": 0, "email": 0, "general": 8, "iot": 0, "lists": 0, "music": 3, "news": 13, "play": 3, "qa": 0, "recommendation": 53, "social": 0, "takeaway": 1, "transport": 4, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 1, "general": 23, "iot": 0, "lists": 0, "music": 0, "news": 4, "play": 0, "qa": 0, "recommendation": 0, "social": 77, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 4, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 48, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 6, "iot": 0, "lists": 0, "music": 0, "news": 2, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 107, "weather": 5, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 1, "general": 6, "iot": 0, "lists": 0, "music": 0, "news": 6, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 135, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 818, "accuracy": 0.6973572037510657, "wilson95": [ 0.6704579471495642, 0.722968029246585 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 818, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6973572037510657, "cluster_bootstrap95": [ 0.6699242645448572, 0.7262143610244696 ], "independent_source_groups": 47, "brier": 0.4108983065122691, "nll": 0.7198321887151691, "ece10": 0.04707003266098168, "calibration_n": 1173, "confident_wrong": 14, "macro_f1": 0.6571650416795958, "confusion_matrix": { "negative": { "negative": 100, "neutral": 68, "positive": 14, "__failed__": 0 }, "neutral": { "negative": 74, "neutral": 468, "positive": 48, "__failed__": 0 }, "positive": { "negative": 28, "neutral": 123, "positive": 250, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "decider-4b": { "label": "Decider 4B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 180, "accuracy": 0.9, "wilson95": [ 0.8505941875672826, 0.9343295513309041 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 180, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.14936982634428456, "nll": 0.2672253166528653, "ece10": 0.03666521824374218, "calibration_n": 200, "confident_wrong": 4, "macro_f1": 0.9025873860182371, "confusion_matrix": { "world": { "world": 46, "sports": 0, "business": 0, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 47, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 40, "sci_tech": 9, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 3, "sci_tech": 47, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 153, "accuracy": 0.765, "wilson95": [ 0.7015863111723746, 0.8184256658476741 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 153, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.765, "brier": 0.3447072046621021, "nll": 0.6337869899393289, "ece10": 0.08634776463492332, "calibration_n": 200, "confident_wrong": 7, "macro_f1": 0.6670168444471164, "confusion_matrix": { "sadness": { "sadness": 49, "joy": 2, "love": 1, "anger": 3, "fear": 2, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 9, "joy": 60, "love": 7, "anger": 1, "fear": 2, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 1, "joy": 5, "love": 6, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 3, "joy": 2, "love": 1, "anger": 21, "fear": 1, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 2, "joy": 1, "love": 0, "anger": 1, "fear": 16, "surprise": 0, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 195, "accuracy": 0.975, "wilson95": [ 0.942821659593358, 0.9892752803482386 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 195, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.975, "brier": 0.044259669099999996, "nll": 0.08873221106055812, "ece10": 0.03234149999999999, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.9747213023585024, "confusion_matrix": { "false": { "false": 108, "true": 1, "__failed__": 0 }, "true": { "false": 4, "true": 87, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": 0.8, "development": { "n": 89, "correct": 89, "wilson95": [ 0.9586234547637477, 1.0 ] }, "test_coverage": 0.935, "accepted_n": 187, "accepted_correct": 186, "accepted_accuracy": 0.9946524064171123, "accepted_wilson95": [ 0.9703356448529545, 0.9990553938825937 ], "errors_per_1000_all_test_cases": 5.0, "note": "Selected on development only, applied unchanged to test. An empirical pilot estimate, not a production guarantee." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 194, "accuracy": 0.97, "wilson95": [ 0.9361057075454807, 0.9861796856598887 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 194, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.97, "brier": 0.05387854090000003, "nll": 0.11085525334152305, "ece10": 0.03743449999999998, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.9684741488020177, "confusion_matrix": { "true": { "true": 75, "false": 2, "__failed__": 0 }, "false": { "true": 4, "false": 119, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": 0.9, "development": { "n": 86, "correct": 86, "wilson95": [ 0.9572418027142576, 0.9999999999999999 ] }, "test_coverage": 0.805, "accepted_n": 161, "accepted_correct": 160, "accepted_accuracy": 0.9937888198757764, "accepted_wilson95": [ 0.9656604381231965, 0.9989027314672388 ], "errors_per_1000_all_test_cases": 5.0, "note": "Selected on development only, applied unchanged to test. An empirical pilot estimate, not a production guarantee." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.2590308295014239, "nll": 0.48597702216517474, "ece10": 0.18675781494816152, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8666666666666666, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.4966458789783434, "nll": 1.0563589140016292, "ece10": 0.3290010543429272, "calibration_n": 12, "confident_wrong": 1, "macro_f1": 0.6733333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 1, "documents": 1, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.05322362848600178, "nll": 0.09067460480508731, "ece10": 0.06763332920008328, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9142857142857143, "confusion_matrix": { "design": { "design": 2, "inspection": 0, "purchase": 1, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.01387509199485657, "nll": 0.06127810845736268, "ece10": 0.0566188619381886, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 96, "accuracy": 0.96, "wilson95": [ 0.9016292856411208, 0.9843366960084523 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 96, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.96, "brier": 0.07362957162304609, "nll": 0.1731274565712879, "ece10": 0.027525224511812225, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.9593627450980392, "confusion_matrix": { "label_000": { "label_000": 24, "label_001": 1, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 22, "label_002": 1, "label_003": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 71, "accuracy": 0.71, "wilson95": [ 0.6146110882442888, 0.7898516425087772 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 71, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.71, "brier": 0.388493113260396, "nll": 0.7644768242365346, "ece10": 0.08683264225443575, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.7075294296346928, "confusion_matrix": { "label_000": { "label_000": 14, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 1, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 15, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 13, "label_003": 2, "label_004": 2, "label_005": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 6, "label_003": 8, "label_004": 3, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 1, "label_001": 2, "label_002": 0, "label_003": 1, "label_004": 12, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 2, "label_002": 2, "label_003": 0, "label_004": 2, "label_005": 9, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 89, "accuracy": 0.89, "wilson95": [ 0.8136870349691969, 0.9374580364293543 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 89, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.89, "brier": 0.17110831972102858, "nll": 0.42087579983977397, "ece10": 0.08608478590209316, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8675925925925925, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 2, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 2, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 2, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 1, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 2, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 4.542341801385178e-05, "nll": 0.0028119026100004323, "ece10": 0.0028000139604376175, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 72, "accuracy": 0.6486486486486487, "wilson95": [ 0.5562227910850616, 0.7311298804925787 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 72, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6486486486486487, "brier": 0.4690111975834351, "nll": 0.8042067931728505, "ece10": 0.17178541552843074, "calibration_n": 111, "confident_wrong": 6 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 71, "accuracy": 0.9861111111111112, "wilson95": [ 0.9254339785252386, 0.9975440481170943 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 71, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9861111111111112, "cluster_bootstrap95": [ 0.9583333333333334, 1.0 ], "independent_source_groups": 36, "brier": 0.0573004574066254, "nll": 0.11998142625509793, "ece10": 0.07507866396596644, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.12008370736842103, "nll": 0.23190707395421536, "ece10": 0.12557894736842104, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9467787114845938, "confusion_matrix": { "false": { "false": 10, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03326607499999999, "nll": 0.1041950103518719, "ece10": 0.094475, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.1320370236363636, "nll": 0.2251478896181906, "ece10": 0.14279999999999998, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8610526315789473, "confusion_matrix": { "false": { "false": 8, "true": 2, "__failed__": 0 }, "true": { "false": 1, "true": 11, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.0529997761190129, "nll": 0.08729013680879835, "ece10": 0.06354576918758267, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9415584415584416, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 1, "feature_request": 0, "praise": 0, "pricing_billing": 3, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.2348691602481407, "nll": 0.469171285373159, "ece10": 0.2803657992154358, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.03363909604529281, "nll": 0.062061371140793936, "ece10": 0.04758932163384054, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 1, "roads": 3, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.19074768949644139, "nll": 0.3565968262893755, "ece10": 0.21741446866221018, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8380952380952381, "confusion_matrix": { "bug": { "bug": 2, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 1, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 1, "security": 0, "style": 1, "refactor": 0, "nit": 2, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.19749348774950223, "nll": 0.3866109689263993, "ece10": 0.09503851681845184, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.837037037037037, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 0, "chore": 1, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.10435926941176468, "nll": 0.18237640718837356, "ece10": 0.13847647058823528, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9403508771929825, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.21076313874999997, "nll": 0.35901612705389924, "ece10": 0.06305625, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.18835195102189004, "nll": 0.3426667488996827, "ece10": 0.10406916656546943, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.877948717948718, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 1, "threat": 1, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 6, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 21, "accuracy": 0.9545454545454546, "wilson95": [ 0.7820195416134267, 0.991930572378736 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9545454545454546, "brier": 0.09455844660959531, "nll": 0.2298371440526613, "ece10": 0.13823210359652033, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 1, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 3, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.009756124718522961, "nll": 0.031084097947356514, "ece10": 0.027972308464252478, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 4.759606184107818e-06, "nll": 0.0014805159781352251, "ece10": 0.0014789010595784506, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.07720123764705883, "nll": 0.1928952880408905, "ece10": 0.16911764705882354, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00023670602011997358, "nll": 0.007638248914075162, "ece10": 0.007546906720269083, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.12271899904097514, "nll": 0.21389689632856365, "ece10": 0.046824708267580395, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.03825675038532993, "nll": 0.08600408872458368, "ece10": 0.07062355212900878, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.23232328555555556, "nll": 0.3547185186274762, "ece10": 0.18032777777777775, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8875, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.36041145975896405, "nll": 0.560467738745163, "ece10": 0.18374598227795982, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.625, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.18865871746054283, "nll": 0.3315062472794905, "ece10": 0.15478646087258682, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.890993265993266, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 18, "accuracy": 0.9, "wilson95": [ 0.6989663547715128, 0.9721335187862319 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.17359122962811468, "nll": 0.34832358069890434, "ece10": 0.16870415588484158, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8918181818181818, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 4, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 1, "smurfing": 2, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 13, "accuracy": 0.6190476190476191, "wilson95": [ 0.4087865377511576, 0.7924898863143551 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6190476190476191, "brier": 0.5723358886442318, "nll": 1.1752435935815844, "ece10": 0.20170448742780678, "calibration_n": 21, "confident_wrong": 1, "macro_f1": 0.6387301587301586, "confusion_matrix": { "spelling": { "spelling": 2, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 2, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 3, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 0, "none": 3, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 2, "none": 3, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03502188250000001, "nll": 0.09779856700303405, "ece10": 0.08658750000000001, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.004311048416722365, "nll": 0.032585000474929265, "ece10": 0.03123326952545742, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 11, "accuracy": 0.7857142857142857, "wilson95": [ 0.524107694133997, 0.9242861328730683 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.2873145543545662, "nll": 0.5141567494772181, "ece10": 0.11981351872563517, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.6611111111111111, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.20736067934289065, "nll": 0.42101503297870124, "ece10": 0.3341175929353634, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.39969761583328955, "nll": 0.719632872146763, "ece10": 0.2469616292984913, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7430555555555555, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 1, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06866437764705884, "nll": 0.17380024120407797, "ece10": 0.15213529411764704, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06713902980987975, "nll": 0.1572293631899691, "ece10": 0.12794928585949283, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.07028161891104785, "nll": 0.1673121589793121, "ece10": 0.11474936048984652, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "payments": { "payments": 3, "web": 1, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.15733607647058825, "nll": 0.25817312500158296, "ece10": 0.10279411764705883, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8819444444444444, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06156039500000001, "nll": 0.16143566565179376, "ece10": 0.14185, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.30966458233530814, "nll": 0.556296178061333, "ece10": 0.1717293305362933, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7261072261072261, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 3, "1": 2, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.07651164347697595, "nll": 0.1471756499782807, "ece10": 0.032755210726940986, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571427, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.1838725411764706, "nll": 0.32957219669066107, "ece10": 0.192, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9411764705882353, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.1912528296245413, "nll": 0.2869424898414396, "ece10": 0.08164275612137199, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.838095238095238, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 0, "changed_mind": 3, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.4448956884716611, "nll": 1.0165137307052943, "ece10": 0.09041940797713852, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6619047619047619, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 2, "3": 2, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.2153789725, "nll": 0.32624728854944546, "ece10": 0.18002500000000002, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8117647058823529, "confusion_matrix": { "false": { "false": 7, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.21315158800000003, "nll": 0.36859543444380305, "ece10": 0.17359333333333338, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8611111111111112, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.09349367796525655, "nll": 0.18419093746544185, "ece10": 0.06615213715287427, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 1, "tech": 0, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.16850345411764703, "nll": 0.29796125198413553, "ece10": 0.12129411764705882, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9411764705882353, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 14, "accuracy": 0.7368421052631579, "wilson95": [ 0.5120844910218489, 0.8819358765321748 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7368421052631579, "brier": 0.31568090551831196, "nll": 0.5440158829703152, "ece10": 0.2992972806193412, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.6091269841269842, "confusion_matrix": { "0": { "0": 5, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.26848168117647064, "nll": 0.435414454503566, "ece10": 0.1856764705882353, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.7733333333333333, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.01717290222222222, "nll": 0.08063411530372848, "ece10": 0.07586666666666667, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 17, "accuracy": 0.85, "wilson95": [ 0.6395811352592431, 0.9476312541037835 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.3153439565929067, "nll": 0.546606154649668, "ece10": 0.27055611161541115, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.840491452991453, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.057675478647135826, "nll": 0.0869253846980769, "ece10": 0.052080129028201264, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 15, "accuracy": 0.75, "wilson95": [ 0.531299122381256, 0.8881382985923343 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.29733358700000007, "nll": 0.46953546176273325, "ece10": 0.14009500000000003, "calibration_n": 20, "confident_wrong": 1, "macro_f1": 0.7442455242966752, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.3199009684338481, "nll": 0.622522628584698, "ece10": 0.34339439302026203, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9075369075369075, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 1, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 531, "accuracy": 0.59, "wilson95": [ 0.5575510431132032, 0.6216839304753868 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 689, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6421248835041938, "cluster_bootstrap95": [ 0.5581390555800533, 0.6296783157389794 ], "independent_source_groups": 596, "brier": 0.4541302985301722, "nll": 0.7985750945642923, "ece10": 0.12163285046547342, "calibration_n": 1073, "confident_wrong": 23 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 415, "accuracy": 0.7229965156794426, "wilson95": [ 0.6849955742967314, 0.7580325194687553 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 761, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.813034188034188, "brier": 0.2815705349646334, "nll": 0.5720039919951198, "ece10": 0.05515638371529349, "calibration_n": 936, "confident_wrong": 37 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 363, "accuracy": 0.5185714285714286, "wilson95": [ 0.4815567977500565, 0.5553833393705441 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 689, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6332720588235294, "cluster_bootstrap95": [ 0.4827064277659441, 0.5546016347013812 ], "independent_source_groups": 650, "brier": 0.4734142326126238, "nll": 0.9069136613279422, "ece10": 0.09852937113144679, "calibration_n": 1088, "confident_wrong": 23 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 66, "accuracy": 0.165, "wilson95": [ 0.13184503236772813, 0.2045282048362506 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1358, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.679, "brier": 0.43375790537089454, "nll": 0.7500949509503808, "ece10": 0.022841324038843997, "calibration_n": 2000, "confident_wrong": 19 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.002443035017188914, "nll": 0.026418564447945107, "ece10": 0.02567461686231758, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.017515484739388985, "nll": 0.04443102190373237, "ece10": 0.03789845844452026, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.027536421381271332, "nll": 0.08300892705578712, "ece10": 0.07345976477438335, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.038471936181069914, "nll": 0.10141068213606642, "ece10": 0.08733822128416792, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.11417147379621843, "nll": 0.2284613998962297, "ece10": 0.09742508042488074, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "review": { "review": 8, "it": 0, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 6, "procurement": 0, "maintenance": 2, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.09570668181008073, "nll": 0.20149211997045033, "ece10": 0.09743075908188258, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9499019607843138, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 7, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 31, "accuracy": 0.775, "wilson95": [ 0.6249690325768998, 0.8768390867645927 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 31, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.775, "brier": 0.29801943132848746, "nll": 0.5362484467044503, "ece10": 0.08653166135631661, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7730175706646296, "confusion_matrix": { "review": { "review": 4, "it": 0, "documents": 1, "procurement": 1, "maintenance": 2, "__failed__": 0 }, "it": { "review": 0, "it": 7, "documents": 0, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 5, "procurement": 1, "maintenance": 2, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 7, "maintenance": 1, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 33, "accuracy": 0.825, "wilson95": [ 0.6805000966821229, 0.9125458625396408 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 33, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.825, "brier": 0.2674864862926518, "nll": 0.4982585462529496, "ece10": 0.14525287826930378, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8206162464985995, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 2, "procurement": 0, "documents": 6, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 7, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 1, "documents": 1, "it": 0, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 326, "accuracy": 0.815, "wilson95": [ 0.7740142283989209, 0.8499930261704722 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 326, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.815, "brier": 0.27361720428582, "nll": 0.5184341138303585, "ece10": 0.038183768491147976, "calibration_n": 400, "confident_wrong": 5, "macro_f1": 0.7513826026830702, "confusion_matrix": { "sadness": { "sadness": 108, "joy": 5, "love": 3, "anger": 1, "fear": 2, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 3, "joy": 99, "love": 10, "anger": 2, "fear": 1, "surprise": 3, "__failed__": 0 }, "love": { "sadness": 3, "joy": 12, "love": 19, "anger": 1, "fear": 2, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 9, "joy": 3, "love": 3, "anger": 46, "fear": 2, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 1, "joy": 0, "love": 0, "anger": 3, "fear": 48, "surprise": 0, "__failed__": 0 }, "surprise": { "sadness": 1, "joy": 0, "love": 1, "anger": 0, "fear": 2, "surprise": 6, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 280, "accuracy": 0.9333333333333333, "wilson95": [ 0.899276940483885, 0.9564324844548417 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 280, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.0854434055089432, "nll": 0.19703224672305356, "ece10": 0.019299206012468602, "calibration_n": 300, "confident_wrong": 1 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 373, "accuracy": 0.9325, "wilson95": [ 0.9035733345645931, 0.9531985308362403 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 373, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9325, "brier": 0.11644558509999996, "nll": 0.19824861145477096, "ece10": 0.04129150000000001, "calibration_n": 400, "confident_wrong": 2, "macro_f1": 0.9057846481318317, "confusion_matrix": { "false": { "false": 293, "true": 16, "__failed__": 0 }, "true": { "false": 11, "true": 80, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 365, "accuracy": 0.9125, "wilson95": [ 0.8807392673857086, 0.9364130897884968 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 365, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9125, "brier": 0.14441686784248928, "nll": 0.2877197236158971, "ece10": 0.03791700308796256, "calibration_n": 400, "confident_wrong": 15, "macro_f1": 0.9045593023278964, "confusion_matrix": { "world": { "world": 89, "sports": 6, "business": 7, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 2, "sports": 121, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 1, "sports": 0, "business": 64, "sci_tech": 7, "__failed__": 0 }, "sci_tech": { "world": 1, "sports": 0, "business": 10, "sci_tech": 91, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 384, "accuracy": 0.96, "wilson95": [ 0.9360177522821589, 0.9752309369302883 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 384, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.96, "brier": 0.0577417122, "nll": 0.1146748095054281, "ece10": 0.043572500000000014, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.9581546186839627, "confusion_matrix": { "true": { "true": 150, "false": 3, "__failed__": 0 }, "false": { "true": 13, "false": 234, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 224, "accuracy": 0.7466666666666667, "wilson95": [ 0.6945458844004171, 0.7925502497954735 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 224, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7466666666666667, "brier": 0.36060334126443594, "nll": 0.8441383054114834, "ece10": 0.05090926074759907, "calibration_n": 300, "confident_wrong": 1 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 395, "accuracy": 0.9875, "wilson95": [ 0.9710762638040326, 0.9946492492200284 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 395, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9875, "brier": 0.02557501679999999, "nll": 0.06748722644266408, "ece10": 0.04384650000000002, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.9874654516511134, "confusion_matrix": { "false": { "false": 187, "true": 1, "__failed__": 0 }, "true": { "false": 4, "true": 208, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 311, "accuracy": 0.7775, "wilson95": [ 0.7342089511162986, 0.8155117255281668 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 311, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7775, "brier": 0.2974494336500002, "nll": 0.4461207815389065, "ece10": 0.03458225, "calibration_n": 400, "confident_wrong": 8, "macro_f1": 0.7710772871372443, "confusion_matrix": { "false": { "false": 189, "true": 11, "__failed__": 0 }, "true": { "false": 78, "true": 122, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 77, "accuracy": 0.77, "wilson95": [ 0.6784561697712622, 0.8415673411969654 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 77, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.77, "brier": 0.38019633060000024, "nll": 0.7054294662419437, "ece10": 0.165157, "calibration_n": 100, "confident_wrong": 12, "macro_f1": 0.7546666666666666, "confusion_matrix": { "yes": { "yes": 51, "no": 3, "__failed__": 0 }, "no": { "yes": 20, "no": 26, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 55, "accuracy": 0.55, "wilson95": [ 0.45244602997442135, 0.6438546202048803 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 55, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.683021029544619, "nll": 1.7268940324457196, "ece10": 0.2796092597390162, "calibration_n": 100, "confident_wrong": 12, "macro_f1": 0.572766167870657, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 4, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 5, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 5, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 2, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 3, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 5, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 5, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 6, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 5, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 4, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 3, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 143, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.715, "brier": 0.34199901662517346, "nll": 0.5998303614981345, "ece10": 0.1025992410804292, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 127, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.635, "brier": 0.5178463052208113, "nll": 1.1335858747484344, "ece10": 0.17548589248720017, "calibration_n": 200, "confident_wrong": 15 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.32339824524164706, "nll": 0.9351002903870537, "ece10": 0.1220870202856873, "calibration_n": 100, "confident_wrong": 8, "macro_f1": 0.7439904132655281, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 13, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 1, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 12, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 8, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 2, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 2, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 5, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 6, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 6, "accuracy": 0.06, "wilson95": [ 0.027786123963188192, 0.1247681544589592 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 211, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5275, "brier": 0.7587081751812476, "nll": 1.842626927412343, "ece10": 0.303207079659769, "calibration_n": 400, "confident_wrong": 71 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 67, "accuracy": 0.67, "wilson95": [ 0.5730534379509052, 0.7543687726587197 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 67, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.67, "brier": 0.5010664064430747, "nll": 1.1480218527841635, "ece10": 0.06996020937069106, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.6685659223817119, "confusion_matrix": { "politics": { "politics": 2, "business": 0, "technology": 1, "science": 5, "health": 3, "sports": 0, "entertainment": 0, "world": 1, "culture": 1, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 1, "technology": 8, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 1, "technology": 0, "science": 8, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 1, "health": 0, "sports": 7, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 0, "technology": 1, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 2, "weather": 0, "__failed__": 0 }, "world": { "politics": 1, "business": 1, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 5, "culture": 1, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 1, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 8, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 0, "entertainment": 0, "world": 2, "culture": 0, "weather": 5, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 41, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.41, "brier": 0.7679095162661327, "nll": 1.8453950857663797, "ece10": 0.1402706679775319, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.38651144513213476, "confusion_matrix": { "biology": { "biology": 8, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 7, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 2, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 1, "medicine": 0, "chemistry": 4, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 3, "medicine": 2, "chemistry": 1, "physics": 4, "computer_science": 0, "mathematics": 0, "earth_science": 3, "engineering": 2, "economics": 1, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 3, "medicine": 2, "chemistry": 2, "physics": 0, "computer_science": 4, "mathematics": 0, "earth_science": 2, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 1, "physics": 2, "computer_science": 1, "mathematics": 2, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 1, "medicine": 3, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 0, "earth_science": 6, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 1, "chemistry": 1, "physics": 0, "computer_science": 0, "mathematics": 2, "earth_science": 0, "engineering": 3, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 3, "other": 0, "__failed__": 0 }, "other": { "biology": 2, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 2, "earth_science": 3, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 15, "accuracy": 0.15, "wilson95": [ 0.0930598528365663, 0.23283559590832364 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 78, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.39, "brier": 0.7009225503861184, "nll": 1.7467405130881166, "ece10": 0.31225609087464085, "calibration_n": 100, "confident_wrong": 27 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 26, "accuracy": 0.26, "wilson95": [ 0.18404698464748132, 0.3537098944918718 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 113, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.565, "brier": 0.2731200126306111, "nll": 0.4820465817573889, "ece10": 0.039472146392111454, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 84, "accuracy": 0.84, "wilson95": [ 0.7557973061072976, 0.8990471151119521 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 84, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.84, "brier": 0.2181520392080814, "nll": 0.4479090782123484, "ece10": 0.0713812505648425, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8400125329984963, "confusion_matrix": { "negative": { "negative": 35, "neutral": 1, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 10, "neutral": 23, "positive": 0, "__failed__": 0 }, "positive": { "negative": 1, "neutral": 4, "positive": 26, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 24, "accuracy": 0.24, "wilson95": [ 0.16691325556948772, 0.33232336349814473 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 107, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.535, "brier": 0.2664394912, "nll": 0.38286832727849757, "ece10": 0.07395399999999999, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 44, "accuracy": 0.44, "wilson95": [ 0.34672026630160796, 0.5377189534832302 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 230, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7666666666666667, "brier": 0.3428816320327836, "nll": 0.7394764964595085, "ece10": 0.1105489813632739, "calibration_n": 300, "confident_wrong": 17 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 65, "accuracy": 0.65, "wilson95": [ 0.5525444329688796, 0.7363575175690247 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 65, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.5419218157498076, "nll": 1.718464325034911, "ece10": 0.17117413170640422, "calibration_n": 100, "confident_wrong": 9, "macro_f1": 0.5912853019995877, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 1, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 5, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 1, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 1, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 1, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 4, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 3, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 2, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 3, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 9, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 5, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 4, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 2, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 33, "accuracy": 0.33, "wilson95": [ 0.24563122734128034, 0.4269465620490947 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 33, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.33, "brier": 1.035786985987986, "nll": 2.6483735299627535, "ece10": 0.4767330801709931, "calibration_n": 100, "confident_wrong": 18, "macro_f1": 0.3169423467810565, "confusion_matrix": { "billing": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 3, "account_access": 5, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 2, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 1, "onboarding": 0, "integrations": 4, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 2, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 5, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 1, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 2, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 2, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 3, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 1, "data_privacy": 2, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 6, "accuracy": 0.06, "wilson95": [ 0.027786123963188192, 0.1247681544589592 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 140, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.7559427059124079, "nll": 1.7224785832189973, "ece10": 0.27908061463103234, "calibration_n": 300, "confident_wrong": 21 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 66, "accuracy": 0.66, "wilson95": [ 0.5627772885472462, 0.7453847920265184 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 66, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.66, "brier": 0.5911071749947043, "nll": 1.6837171079481175, "ece10": 0.23587655755126072, "calibration_n": 100, "confident_wrong": 16, "macro_f1": 0.6461448760790867, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 5, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 4, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 7, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 4, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 1, "loyalty": 7, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 4, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 4, "car": 2, "visa_docs": 1, "loyalty": 1, "disruption": 1, "other": 1, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 96, "accuracy": 0.96, "wilson95": [ 0.9016292856411208, 0.9843366960084523 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 96, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.96, "brier": 0.06766735618440425, "nll": 0.17325129860931449, "ece10": 0.03141272879797728, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.9597794117647059, "confusion_matrix": { "business": { "business": 23, "science and technology": 2, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 0, "science and technology": 23, "sports": 1, "world": 1, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 87, "accuracy": 0.87, "wilson95": [ 0.7901964855992357, 0.9224283257275948 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 87, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.17048129932783432, "nll": 0.37598265254560637, "ece10": 0.042364464888498654, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.850462962962963, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 2, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 2, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 2, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 1, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 2, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 1, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 77, "accuracy": 0.77, "wilson95": [ 0.6784561697712622, 0.8415673411969654 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 77, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.77, "brier": 0.3594046738065104, "nll": 0.714745127750357, "ece10": 0.06789265200667842, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.7656179989513324, "confusion_matrix": { "anger": { "anger": 16, "fear": 1, "joy": 0, "love": 0, "sadness": 0, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 0, "fear": 15, "joy": 0, "love": 0, "sadness": 1, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 12, "love": 2, "sadness": 2, "surprise": 0, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 6, "love": 8, "sadness": 3, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 1, "fear": 2, "joy": 0, "love": 0, "sadness": 13, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 0, "joy": 0, "love": 0, "sadness": 2, "surprise": 13, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2603, "accuracy": 0.8829715061058344, "wilson95": [ 0.8708660763602786, 0.8940801551700913 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2603, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8829715061058344, "brier": 0.18140633564541897, "nll": 0.4231795863527515, "ece10": 0.029058173191562158, "calibration_n": 2948, "confident_wrong": 42, "macro_f1": 0.8799751975441833, "confusion_matrix": { "alarm": { "alarm": 92, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 0, "audio": 51, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 8, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 1, "audio": 0, "calendar": 375, "cooking": 0, "datetime": 4, "email": 0, "general": 5, "iot": 0, "lists": 2, "music": 0, "news": 3, "play": 1, "qa": 4, "recommendation": 3, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 68, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 97, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 242, "general": 16, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 7, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 7, "datetime": 0, "email": 0, "general": 103, "iot": 2, "lists": 0, "music": 1, "news": 5, "play": 4, "qa": 41, "recommendation": 10, "social": 1, "takeaway": 0, "transport": 3, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 2, "calendar": 0, "cooking": 2, "datetime": 0, "email": 0, "general": 8, "iot": 199, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 2, "lists": 123, "music": 4, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 69, "news": 0, "play": 6, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 112, "play": 0, "qa": 9, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 10, "iot": 2, "lists": 0, "music": 29, "news": 3, "play": 336, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 9, "iot": 0, "lists": 0, "music": 1, "news": 7, "play": 0, "qa": 263, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 2, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 2, "play": 4, "qa": 6, "recommendation": 76, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 19, "iot": 1, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 1, "recommendation": 0, "social": 83, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 0, "email": 0, "general": 1, "iot": 1, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 52, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 5, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 116, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 146, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2418, "accuracy": 0.8202170963364993, "wilson95": [ 0.8059412231536095, 0.8336595231392775 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2418, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8202170963364993, "brier": 0.25970484488224976, "nll": 0.6124326216724015, "ece10": 0.03533059669226825, "calibration_n": 2948, "confident_wrong": 35, "macro_f1": 0.8157215631118525, "confusion_matrix": { "alarm": { "alarm": 88, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 44, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 10, "iot": 2, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 0, "audio": 1, "calendar": 359, "cooking": 0, "datetime": 7, "email": 1, "general": 10, "iot": 6, "lists": 2, "music": 0, "news": 1, "play": 1, "qa": 4, "recommendation": 5, "social": 1, "takeaway": 1, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 66, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 99, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 236, "general": 21, "iot": 1, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 5, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 10, "cooking": 6, "datetime": 0, "email": 1, "general": 100, "iot": 2, "lists": 0, "music": 1, "news": 6, "play": 3, "qa": 35, "recommendation": 12, "social": 0, "takeaway": 0, "transport": 7, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 6, "calendar": 0, "cooking": 3, "datetime": 1, "email": 0, "general": 14, "iot": 178, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 6, "transport": 1, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 1, "calendar": 2, "cooking": 0, "datetime": 0, "email": 1, "general": 9, "iot": 5, "lists": 116, "music": 3, "news": 0, "play": 2, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 66, "news": 0, "play": 4, "qa": 4, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 3, "iot": 1, "lists": 1, "music": 0, "news": 107, "play": 0, "qa": 7, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 0, "weather": 2, "__failed__": 0 }, "play": { "alarm": 1, "audio": 9, "calendar": 0, "cooking": 2, "datetime": 5, "email": 1, "general": 17, "iot": 2, "lists": 5, "music": 66, "news": 3, "play": 265, "qa": 8, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 16, "iot": 0, "lists": 0, "music": 1, "news": 16, "play": 1, "qa": 248, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 2, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 5, "qa": 6, "recommendation": 75, "social": 0, "takeaway": 2, "transport": 2, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 3, "general": 21, "iot": 2, "lists": 0, "music": 0, "news": 2, "play": 0, "qa": 1, "recommendation": 0, "social": 72, "takeaway": 2, "transport": 0, "weather": 1, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 2, "datetime": 0, "email": 0, "general": 3, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 48, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 2, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 111, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 2, "weather": 140, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 850, "accuracy": 0.7246376811594203, "wilson95": [ 0.6983726416057693, 0.7494361911388755 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 850, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7246376811594203, "cluster_bootstrap95": [ 0.6922916666666666, 0.7555950877241686 ], "independent_source_groups": 47, "brier": 0.39134169197862184, "nll": 0.6830708612052179, "ece10": 0.04926853062606126, "calibration_n": 1173, "confident_wrong": 8, "macro_f1": 0.6875618294820401, "confusion_matrix": { "negative": { "negative": 106, "neutral": 66, "positive": 10, "__failed__": 0 }, "neutral": { "negative": 66, "neutral": 483, "positive": 41, "__failed__": 0 }, "positive": { "negative": 24, "neutral": 116, "positive": 261, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "decision-4b-v12": { "label": "Decision 4B v1.2", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 174, "accuracy": 0.87, "wilson95": [ 0.8163364812775159, 0.9097179772033067 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 174, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.19271580392050094, "nll": 0.3656381443936775, "ece10": 0.04849990651739082, "calibration_n": 200, "confident_wrong": 2, "macro_f1": 0.8739023046466459, "confusion_matrix": { "world": { "world": 43, "sports": 0, "business": 3, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 45, "business": 2, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 47, "sci_tech": 2, "__failed__": 0 }, "sci_tech": { "world": 1, "sports": 0, "business": 12, "sci_tech": 39, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 116, "accuracy": 0.58, "wilson95": [ 0.5107207393958046, 0.6462640083838327 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 116, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.6205005998158565, "nll": 1.2846094317387706, "ece10": 0.1117236836883964, "calibration_n": 200, "confident_wrong": 8, "macro_f1": 0.4485503603150662, "confusion_matrix": { "sadness": { "sadness": 44, "joy": 3, "love": 2, "anger": 3, "fear": 5, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 18, "joy": 42, "love": 12, "anger": 3, "fear": 4, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 2, "joy": 3, "love": 7, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 8, "joy": 3, "love": 0, "anger": 15, "fear": 1, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 7, "joy": 2, "love": 1, "anger": 1, "fear": 8, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 189, "accuracy": 0.945, "wilson95": [ 0.9042130012276914, 0.9690146582965411 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 189, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.945, "brier": 0.16393106635094795, "nll": 0.30470643518061163, "ece10": 0.1814526278641094, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.944688874921433, "confusion_matrix": { "false": { "false": 102, "true": 7, "__failed__": 0 }, "true": { "false": 4, "true": 87, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 185, "accuracy": 0.925, "wilson95": [ 0.8799563897637055, 0.954025082815618 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 185, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.1284540315939143, "nll": 0.2397059175017647, "ece10": 0.09309721242197533, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.9213651018321931, "confusion_matrix": { "true": { "true": 71, "false": 6, "__failed__": 0 }, "false": { "true": 9, "false": 114, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.17493570200414477, "nll": 0.39123144405930804, "ece10": 0.20101392016935465, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 0, "review": 3, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3631890986140356, "nll": 0.7269005339520574, "ece10": 0.22392061268506688, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.7742857142857142, "confusion_matrix": { "maintenance": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 1, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.019586546908580915, "nll": 0.09540464475510141, "ece10": 0.08826236832176042, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.012896723505671994, "nll": 0.07221187029161713, "ece10": 0.0672978635296887, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 91, "accuracy": 0.91, "wilson95": [ 0.837737871472837, 0.9519274599974348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 91, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.91, "brier": 0.1396569622800377, "nll": 0.2763374354010179, "ece10": 0.0768185333839802, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.9038188915340997, "confusion_matrix": { "label_000": { "label_000": 25, "label_001": 0, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 6, "label_001": 16, "label_002": 1, "label_003": 2, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 51, "accuracy": 0.51, "wilson95": [ 0.4134801193402717, 0.6057800106955886 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 51, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.51, "brier": 0.6495110817302268, "nll": 1.331294047617886, "ece10": 0.1306594333646446, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5143459316093587, "confusion_matrix": { "label_000": { "label_000": 9, "label_001": 2, "label_002": 1, "label_003": 1, "label_004": 3, "label_005": 1, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 12, "label_002": 0, "label_003": 0, "label_004": 3, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 9, "label_003": 1, "label_004": 4, "label_005": 2, "__failed__": 0 }, "label_003": { "label_000": 1, "label_001": 0, "label_002": 4, "label_003": 5, "label_004": 7, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 2, "label_002": 2, "label_003": 3, "label_004": 9, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 1, "label_002": 3, "label_003": 0, "label_004": 4, "label_005": 7, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "label_000": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_001": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.004288531262577542, "nll": 0.02554107719169546, "ece10": 0.02422241136357079, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 86, "accuracy": 0.7747747747747747, "wilson95": [ 0.6886323576918694, 0.8425347018910412 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7747747747747747, "brier": 0.30985415130375693, "nll": 0.5541148382144199, "ece10": 0.04404721565378169, "calibration_n": 111, "confident_wrong": 2 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 69, "accuracy": 0.9583333333333334, "wilson95": [ 0.8845072325868389, 0.9857291925330748 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9583333333333334, "cluster_bootstrap95": [ 0.8888888888888888, 1.0 ], "independent_source_groups": 36, "brier": 0.06868523296357797, "nll": 0.15869563532130238, "ece10": 0.09874666048447595, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.10126093947557475, "nll": 0.2098124691508951, "ece10": 0.12427548372869386, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9449275362318841, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03737371097586665, "nll": 0.11987069548889621, "ece10": 0.10880588748932146, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 18, "accuracy": 0.8181818181818182, "wilson95": [ 0.6148339256287345, 0.9269311541657794 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8181818181818182, "brier": 0.26908542088007664, "nll": 0.42150196924041816, "ece10": 0.12078664642041037, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8035714285714286, "confusion_matrix": { "false": { "false": 6, "true": 4, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.03444064529127385, "nll": 0.07926855102349606, "ece10": 0.061688703720761004, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9415584415584416, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 1, "feature_request": 0, "praise": 0, "pricing_billing": 3, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.15469997941271577, "nll": 0.334193853826636, "ece10": 0.18563378933187114, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.08602437974524758, "nll": 0.14499847063411583, "ece10": 0.06680800869287253, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 1, "roads": 3, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.21252982006382595, "nll": 0.41311163567286197, "ece10": 0.19032397478361948, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8650793650793651, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 1, "refactor": 1, "nit": 2, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.2208577610876013, "nll": 0.4457496396925062, "ece10": 0.20151844977627154, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8777777777777778, "confusion_matrix": { "feature": { "feature": 2, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 1, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 0, "chore": 1, "__failed__": 0 }, "test": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 2, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.09218587926029985, "nll": 0.2031676348347989, "ece10": 0.1333451459775349, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9403508771929825, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.1998269195851593, "nll": 0.35641527617435304, "ece10": 0.1395244793017128, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.18758036404771417, "nll": 0.385986789307432, "ece10": 0.2159611114552259, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8377777777777778, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 1, "threat": 1, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 1, "none": 5, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.16110941978878457, "nll": 0.3318736332847772, "ece10": 0.12113872697565864, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8565656565656565, "confusion_matrix": { "news_report": { "news_report": 4, "opinion": 0, "press_release": 1, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 2, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 2, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.004274421652524521, "nll": 0.02702069888677816, "ece10": 0.02557300256807577, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0017336809968674974, "nll": 0.025898633427392716, "ece10": 0.0252579475009898, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06993470230856835, "nll": 0.18722550283641526, "ece10": 0.1659462237591129, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.01336230496264272, "nll": 0.057622127933785426, "ece10": 0.052322179646576525, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.12468486898436197, "nll": 0.23401793075146676, "ece10": 0.1588072943118379, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.09065102369426412, "nll": 0.18455852457801003, "ece10": 0.10607955172518263, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 4, "office_supplies": 0, "software": 0, "client_entertainment": 1, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 3, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.12545051388524323, "nll": 0.25151507917099875, "ece10": 0.15289222096657745, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9442724458204335, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.3780255978477898, "nll": 0.6270711745107698, "ece10": 0.2052875343550836, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7003968253968254, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 4, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.15500345633134352, "nll": 0.3298185089268437, "ece10": 0.2087728576055185, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9407407407407407, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 16, "accuracy": 0.8, "wilson95": [ 0.5839825677481064, 0.919342337420202 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.22488471578128766, "nll": 0.4313799235269312, "ece10": 0.18514270145910405, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7963858363858364, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 2, "boosting": 0, "smurfing": 0, "other": 2, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 1, "harassment": 0, "boosting": 0, "smurfing": 2, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 10, "accuracy": 0.47619047619047616, "wilson95": [ 0.2834400610153417, 0.6763046541715557 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47619047619047616, "brier": 0.6582086523439717, "nll": 1.1802363100674875, "ece10": 0.2161240576061731, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.4473015873015873, "confusion_matrix": { "spelling": { "spelling": 1, "subject_verb_agreement": 0, "word_choice": 2, "punctuation": 1, "none": 0, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 2, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 2, "none": 1, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 2, "none": 3, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.027136456326427805, "nll": 0.11495330814037236, "ece10": 0.10742139714162738, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0021615647434095925, "nll": 0.02848758174767702, "ece10": 0.027653729711570547, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 12, "accuracy": 0.8571428571428571, "wilson95": [ 0.6005862050302965, 0.9599060787285351 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.21838884645408316, "nll": 0.543345643627748, "ece10": 0.10431225931405656, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.75, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.20864471726387504, "nll": 0.37416948277296946, "ece10": 0.16166060970233764, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8875, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.2888062130542618, "nll": 0.5377776503063474, "ece10": 0.19567316257754336, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8115079365079365, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.21606526477895283, "nll": 0.34134383258726847, "ece10": 0.11014193198906547, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8210526315789474, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.05867434684341966, "nll": 0.13992961504844503, "ece10": 0.10297700994345332, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9377777777777778, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "sports": 2, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03882440079031819, "nll": 0.13653855124255487, "ece10": 0.12169328492734695, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.22201310861291598, "nll": 0.3517551286182104, "ece10": 0.1163789158269135, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8131868131868132, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.08181725163518404, "nll": 0.2109937056695004, "ece10": 0.18632715733631267, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.219536834247811, "nll": 0.4001171957312592, "ece10": 0.21475706108689058, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8771043771043772, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.08988768892088812, "nll": 0.20971519998451107, "ece10": 0.09055356823536041, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8743386243386243, "confusion_matrix": { "italian": { "italian": 1, "french": 1, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.2701519898160582, "nll": 0.4287970816356827, "ece10": 0.1932856584206823, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8819444444444444, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.08080123895381558, "nll": 0.16229365402552193, "ece10": 0.10957090137757698, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8833333333333332, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 2, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 11, "accuracy": 0.6111111111111112, "wilson95": [ 0.3861904160223791, 0.7969475342783638 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6111111111111112, "brier": 0.46674737193429877, "nll": 0.9167463337741663, "ece10": 0.13947880588259884, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6033333333333333, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 1, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 1, "1": 1, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 2, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.31435277353307534, "nll": 0.4661267886645855, "ece10": 0.3168840559507331, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8117647058823529, "confusion_matrix": { "false": { "false": 7, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.21000845826222186, "nll": 0.3417290988449203, "ece10": 0.178352142949443, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9333333333333333, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.1251422130000212, "nll": 0.24290672930862148, "ece10": 0.11091443470227842, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 1, "tech": 0, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.10294472620576489, "nll": 0.2361498174274597, "ece10": 0.20392677289835331, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 12, "accuracy": 0.631578947368421, "wilson95": [ 0.41039533164803066, 0.8085048725486492 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.631578947368421, "brier": 0.45770743436278305, "nll": 0.7685223957870676, "ece10": 0.2546422162893507, "calibration_n": 19, "confident_wrong": 1, "macro_f1": 0.5125, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.2283176121074367, "nll": 0.40020775474362874, "ece10": 0.3215931135696064, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04614600043468682, "nll": 0.13459045625452173, "ece10": 0.11994470662808569, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 13, "accuracy": 0.65, "wilson95": [ 0.43285427668523624, 0.818808175898918 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.3785274548053252, "nll": 0.6591462607811669, "ece10": 0.175424672463622, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.5226190476190476, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 2, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 3, "2": 0, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.07885462828309159, "nll": 0.1467468420444801, "ece10": 0.06466870837945492, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 15, "accuracy": 0.75, "wilson95": [ 0.531299122381256, 0.8881382985923343 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.33769376922960703, "nll": 0.5205761012130445, "ece10": 0.12045856614227889, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7493734335839599, "confusion_matrix": { "false": { "false": 7, "true": 2, "__failed__": 0 }, "true": { "false": 3, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.18064050783827224, "nll": 0.38421618387327783, "ece10": 0.0989882349682312, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9142857142857143, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 2, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 578, "accuracy": 0.6422222222222222, "wilson95": [ 0.6103618189009155, 0.6728736949180913 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 733, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6831314072693383, "cluster_bootstrap95": [ 0.6038746159324917, 0.6862759558100526 ], "independent_source_groups": 596, "brier": 0.442576545446141, "nll": 0.8081844431721461, "ece10": 0.029129029067643333, "calibration_n": 1073, "confident_wrong": 19 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 413, "accuracy": 0.7195121951219512, "wilson95": [ 0.6813952742971592, 0.7547105055032415 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 762, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8141025641025641, "brier": 0.27543911218800954, "nll": 0.5448579965461835, "ece10": 0.06202986733615816, "calibration_n": 936, "confident_wrong": 6 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 364, "accuracy": 0.52, "wilson95": [ 0.48298162382637383, 0.5568000623035038 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 672, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6176470588235294, "cluster_bootstrap95": [ 0.48226824248660755, 0.5563382793192824 ], "independent_source_groups": 650, "brier": 0.5164728717835994, "nll": 0.9833506223672813, "ece10": 0.054197485979661834, "calibration_n": 1088, "confident_wrong": 23 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 69, "accuracy": 0.1725, "wilson95": [ 0.13863520728428091, 0.2125953455046833 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1339, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6695, "brier": 0.4440103899233121, "nll": 0.784806410568105, "ece10": 0.03939426491810319, "calibration_n": 2000, "confident_wrong": 8 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.06505389099787962, "nll": 0.1410168183222163, "ece10": 0.07645552228846526, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 6, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03323057110935749, "nll": 0.11442231382920387, "ece10": 0.10107103514261702, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 24, "accuracy": 0.8571428571428571, "wilson95": [ 0.685101978364747, 0.9430099468025059 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 24, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.12982014951637594, "nll": 0.24164668478150717, "ece10": 0.14752345502155378, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.857031857031857, "confusion_matrix": { "design": { "design": 4, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 6, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06030682799259469, "nll": 0.17448151032375234, "ece10": 0.14910478189372475, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06671999795145353, "nll": 0.2014484448794962, "ece10": 0.17026321017468848, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "review": { "review": 8, "it": 0, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0484070559725089, "nll": 0.14422502408255594, "ece10": 0.11919670585497488, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 8, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 37, "accuracy": 0.925, "wilson95": [ 0.8013576647568946, 0.9741639742254119 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.13800640803550965, "nll": 0.3314609813096577, "ece10": 0.21250798619556757, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9241083099906628, "confusion_matrix": { "review": { "review": 6, "it": 0, "documents": 2, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 7, "documents": 0, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.07938797913878749, "nll": 0.20856698382298675, "ece10": 0.1283838160655451, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9498039215686275, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 7, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 217, "accuracy": 0.5425, "wilson95": [ 0.4935050345733776, 0.5906864204082072 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 217, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5425, "brier": 0.5730143635356975, "nll": 1.169667968235174, "ece10": 0.10043742336344573, "calibration_n": 400, "confident_wrong": 3, "macro_f1": 0.465347876863839, "confusion_matrix": { "sadness": { "sadness": 91, "joy": 7, "love": 9, "anger": 3, "fear": 8, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 30, "joy": 57, "love": 23, "anger": 1, "fear": 3, "surprise": 4, "__failed__": 0 }, "love": { "sadness": 12, "joy": 5, "love": 17, "anger": 1, "fear": 2, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 26, "joy": 2, "love": 5, "anger": 24, "fear": 4, "surprise": 3, "__failed__": 0 }, "fear": { "sadness": 20, "joy": 1, "love": 1, "anger": 3, "fear": 26, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 4, "joy": 1, "love": 1, "anger": 1, "fear": 1, "surprise": 2, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 274, "accuracy": 0.9133333333333333, "wilson95": [ 0.8760440413057516, 0.9401711024819569 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 274, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9133333333333333, "brier": 0.14898948598268777, "nll": 0.394758967474729, "ece10": 0.09119024581644965, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 335, "accuracy": 0.8375, "wilson95": [ 0.7981666097704109, 0.870412591553939 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 335, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8375, "brier": 0.24252454582754976, "nll": 0.38658341924583256, "ece10": 0.08195453256246307, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.8041800037657691, "confusion_matrix": { "false": { "false": 250, "true": 59, "__failed__": 0 }, "true": { "false": 6, "true": 85, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 339, "accuracy": 0.8475, "wilson95": [ 0.8089761434535727, 0.8794128119840912 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 339, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8475, "brier": 0.23554084384451804, "nll": 0.4605522413621908, "ece10": 0.0396455980212924, "calibration_n": 400, "confident_wrong": 17, "macro_f1": 0.8349751585267926, "confusion_matrix": { "world": { "world": 86, "sports": 6, "business": 10, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 3, "sports": 120, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 67, "sci_tech": 3, "__failed__": 0 }, "sci_tech": { "world": 6, "sports": 0, "business": 30, "sci_tech": 66, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 366, "accuracy": 0.915, "wilson95": [ 0.8835678157264721, 0.9385369799760619 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 366, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.915, "brier": 0.15509438873979237, "nll": 0.2724756975171345, "ece10": 0.08489321181992006, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.909804753820034, "confusion_matrix": { "true": { "true": 135, "false": 18, "__failed__": 0 }, "false": { "true": 16, "false": 231, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 235, "accuracy": 0.7833333333333333, "wilson95": [ 0.7332901513759227, 0.8262121649301679 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 235, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7833333333333333, "brier": 0.3386763021500235, "nll": 0.8505691842749937, "ece10": 0.12276497419019854, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 381, "accuracy": 0.9525, "wilson95": [ 0.9270084505901734, 0.9693829230372881 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 381, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9525, "brier": 0.14264957421144456, "nll": 0.2813843874470209, "ece10": 0.17784301863001192, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.9523137254901961, "confusion_matrix": { "false": { "false": 178, "true": 10, "__failed__": 0 }, "true": { "false": 9, "true": 203, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 309, "accuracy": 0.7725, "wilson95": [ 0.7289390669951599, 0.8108767325926485 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 309, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7725, "brier": 0.3194297127314642, "nll": 0.48991609652261525, "ece10": 0.10243240575068985, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.7670832293526322, "confusion_matrix": { "false": { "false": 185, "true": 15, "__failed__": 0 }, "true": { "false": 76, "true": 124, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 70, "accuracy": 0.7, "wilson95": [ 0.6041514536665332, 0.7810511470506724 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 70, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.4392471810492279, "nll": 0.6651493888016314, "ece10": 0.12463927208851136, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.66078697421981, "confusion_matrix": { "yes": { "yes": 52, "no": 2, "__failed__": 0 }, "no": { "yes": 28, "no": 18, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.6679377462330731, "nll": 1.6308068073038111, "ece10": 0.12966155888505998, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5053701739866601, "confusion_matrix": { "balance": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 2, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 2, "dispute_charge": 2, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 3, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 6, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 2, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 2, "statement": 5, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 3, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 4, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 6, "loan_payment": 1, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 4, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 5, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 3, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 4, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 145, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.725, "brier": 0.3426600770861076, "nll": 0.5743841967307035, "ece10": 0.09321260537145559, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 39, "accuracy": 0.39, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 121, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.605, "brier": 0.5022928641618966, "nll": 0.9817847592979237, "ece10": 0.052291570239037584, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 82, "accuracy": 0.82, "wilson95": [ 0.7333264161369926, 0.8829977450105364 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 82, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.82, "brier": 0.2895996884325818, "nll": 0.7387123319930196, "ece10": 0.12133795767628622, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.7888486283460319, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 14, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 12, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 1, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 6, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 2, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 9, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 6, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 6, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 1, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 10, "accuracy": 0.1, "wilson95": [ 0.0552291370606751, 0.17436566150491345 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 216, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.54, "brier": 0.6018924803720023, "nll": 1.1720684696661665, "ece10": 0.09359005850084903, "calibration_n": 400, "confident_wrong": 7 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 64, "accuracy": 0.64, "wilson95": [ 0.5423540245160874, 0.7272877959859567 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 64, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.64, "brier": 0.514027740922849, "nll": 1.2124688732450053, "ece10": 0.04114589661205548, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6389299833841495, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 2, "science": 3, "health": 3, "sports": 0, "entertainment": 1, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 8, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 2, "technology": 8, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 2, "technology": 1, "science": 6, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 6, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 0, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 7, "world": 0, "culture": 1, "weather": 0, "__failed__": 0 }, "world": { "politics": 0, "business": 4, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 3, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 2, "world": 0, "culture": 6, "weather": 1, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 6, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 41, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.41, "brier": 0.7654552666093228, "nll": 1.8338712707112614, "ece10": 0.09440853913255898, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.38198060109824816, "confusion_matrix": { "biology": { "biology": 5, "medicine": 0, "chemistry": 2, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 9, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 5, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 1, "medicine": 4, "chemistry": 1, "physics": 6, "computer_science": 0, "mathematics": 0, "earth_science": 4, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 3, "chemistry": 2, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 4, "engineering": 3, "economics": 1, "other": 1, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 1, "chemistry": 2, "physics": 1, "computer_science": 0, "mathematics": 1, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 1, "medicine": 3, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 0, "earth_science": 6, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 0, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 0, "economics": 4, "other": 0, "__failed__": 0 }, "other": { "biology": 2, "medicine": 1, "chemistry": 0, "physics": 2, "computer_science": 0, "mathematics": 1, "earth_science": 3, "engineering": 2, "economics": 1, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 71, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.355, "brier": 0.5087346792781847, "nll": 0.9950285085862289, "ece10": 0.1406627274214585, "calibration_n": 100, "confident_wrong": 5 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 30, "accuracy": 0.3, "wilson95": [ 0.2189488529493276, 0.3958485463334666 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 122, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.31616462638739096, "nll": 0.5567932256162007, "ece10": 0.12538746134486944, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.21339208673907073, "nll": 0.4121564218773346, "ece10": 0.12302473679789046, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8816602316602317, "confusion_matrix": { "negative": { "negative": 32, "neutral": 4, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 2, "neutral": 31, "positive": 0, "__failed__": 0 }, "positive": { "negative": 0, "neutral": 6, "positive": 25, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 89, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.445, "brier": 0.28526091182128216, "nll": 0.4453081168538023, "ece10": 0.12120347887626494, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 46, "accuracy": 0.46, "wilson95": [ 0.36560808309024106, 0.5573513967663177 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 237, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.30862154087695925, "nll": 0.5875833896327759, "ece10": 0.034488008882203776, "calibration_n": 300, "confident_wrong": 4 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "order_status": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "refund_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 6 }, "shipping_delay": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "pricing_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "speak_to_human": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 7 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 9 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 7 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "promo_code": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "partner_inquiry": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 34, "accuracy": 0.34, "wilson95": [ 0.25461520797348164, 0.43722271145275377 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.34, "brier": 0.7886412782347036, "nll": 1.8779597674126949, "ece10": 0.1784264373499074, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.3131928910808221, "confusion_matrix": { "billing": { "billing": 3, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 6, "shipping": 0, "returns": 0, "technical_outage": 1, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 1, "shipping": 2, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 3, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 3, "shipping": 0, "returns": 0, "technical_outage": 1, "onboarding": 0, "integrations": 3, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 1, "onboarding": 2, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 6, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 2, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 1, "pricing": 1, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 5, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "other": { "billing": 1, "account_access": 0, "shipping": 1, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 2, "compliance": 0, "pricing": 2, "data_privacy": 2, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 143, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4766666666666667, "brier": 0.6405890630213142, "nll": 1.4070234423083001, "ece10": 0.08208717588059117, "calibration_n": 300, "confident_wrong": 0 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.5984387019787392, "nll": 1.3599164173370932, "ece10": 0.15898575676161564, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.5645336145336146, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 4, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 1, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 4, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 11, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 2, "loyalty": 2, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 4, "car": 0, "visa_docs": 1, "loyalty": 3, "disruption": 1, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 5, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 2, "baggage": 1, "refund": 0, "hotel": 4, "car": 2, "visa_docs": 1, "loyalty": 1, "disruption": 0, "other": 1, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.16483188718458253, "nll": 0.31746155844224544, "ece10": 0.07347988933643751, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8687660949623819, "confusion_matrix": { "business": { "business": 25, "science and technology": 0, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 9, "science and technology": 13, "sports": 0, "world": 3, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "refund not showing up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "activating a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 54, "accuracy": 0.54, "wilson95": [ 0.44264860323368227, 0.6343919169097589 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 54, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.54, "brier": 0.6063143427809092, "nll": 1.2470840666418324, "ece10": 0.07148605997825833, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5391269407394148, "confusion_matrix": { "anger": { "anger": 12, "fear": 3, "joy": 0, "love": 1, "sadness": 1, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 1, "fear": 14, "joy": 0, "love": 0, "sadness": 2, "surprise": 0, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 7, "love": 5, "sadness": 3, "surprise": 1, "__failed__": 0 }, "love": { "anger": 1, "fear": 1, "joy": 2, "love": 6, "sadness": 7, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 2, "joy": 2, "love": 3, "sadness": 9, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 1, "joy": 2, "love": 1, "sadness": 5, "surprise": 6, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2425, "accuracy": 0.8225915875169606, "wilson95": [ 0.808384378706726, 0.8359591697302817 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2425, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8225915875169606, "brier": 0.2552732016353492, "nll": 0.6181616741731591, "ece10": 0.05381278709469925, "calibration_n": 2948, "confident_wrong": 27, "macro_f1": 0.8228500024039003, "confusion_matrix": { "alarm": { "alarm": 91, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 48, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 7, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 1, "audio": 0, "calendar": 346, "cooking": 0, "datetime": 9, "email": 0, "general": 10, "iot": 0, "lists": 5, "music": 0, "news": 3, "play": 0, "qa": 6, "recommendation": 17, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 67, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 89, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 11, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 1, "calendar": 5, "cooking": 0, "datetime": 0, "email": 221, "general": 28, "iot": 1, "lists": 9, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 2, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 10, "cooking": 5, "datetime": 0, "email": 0, "general": 117, "iot": 2, "lists": 0, "music": 1, "news": 0, "play": 3, "qa": 23, "recommendation": 16, "social": 1, "takeaway": 2, "transport": 3, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 3, "calendar": 1, "cooking": 4, "datetime": 1, "email": 0, "general": 8, "iot": 188, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 8, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 1, "calendar": 3, "cooking": 0, "datetime": 0, "email": 0, "general": 9, "iot": 1, "lists": 121, "music": 5, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 1, "music": 70, "news": 0, "play": 2, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 0, "general": 11, "iot": 0, "lists": 0, "music": 0, "news": 88, "play": 1, "qa": 17, "recommendation": 1, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "play": { "alarm": 0, "audio": 6, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 24, "iot": 0, "lists": 1, "music": 63, "news": 1, "play": 284, "qa": 2, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 28, "iot": 1, "lists": 0, "music": 3, "news": 1, "play": 1, "qa": 238, "recommendation": 6, "social": 0, "takeaway": 0, "transport": 4, "weather": 4, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 1, "lists": 0, "music": 0, "news": 1, "play": 1, "qa": 5, "recommendation": 83, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 31, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 68, "takeaway": 2, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 2, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 51, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 6, "social": 0, "takeaway": 0, "transport": 114, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 141, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2150, "accuracy": 0.7293080054274084, "wilson95": [ 0.712978241741253, 0.7450409367481446 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2150, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7293080054274084, "brier": 0.3612933041964014, "nll": 0.9067338646636933, "ece10": 0.04952764047939709, "calibration_n": 2948, "confident_wrong": 19, "macro_f1": 0.7360007607799655, "confusion_matrix": { "alarm": { "alarm": 85, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 1, "email": 0, "general": 2, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 2, "audio": 41, "calendar": 1, "cooking": 0, "datetime": 3, "email": 0, "general": 11, "iot": 1, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 3, "audio": 0, "calendar": 326, "cooking": 1, "datetime": 8, "email": 1, "general": 24, "iot": 2, "lists": 5, "music": 0, "news": 2, "play": 0, "qa": 2, "recommendation": 18, "social": 1, "takeaway": 4, "transport": 2, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 64, "datetime": 0, "email": 0, "general": 2, "iot": 1, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 90, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 7, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 0, "email": 215, "general": 31, "iot": 2, "lists": 4, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 1, "social": 3, "takeaway": 0, "transport": 2, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 11, "cooking": 5, "datetime": 2, "email": 1, "general": 106, "iot": 2, "lists": 2, "music": 1, "news": 1, "play": 3, "qa": 28, "recommendation": 13, "social": 0, "takeaway": 0, "transport": 6, "weather": 5, "__failed__": 0 }, "iot": { "alarm": 2, "audio": 7, "calendar": 1, "cooking": 7, "datetime": 1, "email": 0, "general": 9, "iot": 170, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 12, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 1, "datetime": 0, "email": 0, "general": 10, "iot": 3, "lists": 116, "music": 5, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 7, "iot": 0, "lists": 2, "music": 61, "news": 0, "play": 3, "qa": 3, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "news": { "alarm": 0, "audio": 1, "calendar": 3, "cooking": 0, "datetime": 0, "email": 0, "general": 21, "iot": 0, "lists": 2, "music": 0, "news": 70, "play": 0, "qa": 17, "recommendation": 5, "social": 1, "takeaway": 0, "transport": 2, "weather": 2, "__failed__": 0 }, "play": { "alarm": 1, "audio": 15, "calendar": 1, "cooking": 0, "datetime": 5, "email": 0, "general": 55, "iot": 4, "lists": 8, "music": 122, "news": 0, "play": 161, "qa": 5, "recommendation": 5, "social": 0, "takeaway": 1, "transport": 2, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 41, "iot": 3, "lists": 2, "music": 2, "news": 1, "play": 1, "qa": 225, "recommendation": 5, "social": 0, "takeaway": 0, "transport": 2, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 3, "recommendation": 83, "social": 0, "takeaway": 1, "transport": 2, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 3, "general": 29, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 4, "social": 60, "takeaway": 5, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 2, "datetime": 0, "email": 0, "general": 6, "iot": 2, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 43, "transport": 1, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 1, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 5, "social": 0, "takeaway": 1, "transport": 111, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 2, "email": 0, "general": 12, "iot": 5, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 1, "weather": 123, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 787, "accuracy": 0.670929241261722, "wilson95": [ 0.6435199563115449, 0.6972226279969987 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 787, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.670929241261722, "cluster_bootstrap95": [ 0.6344639440010935, 0.7062618074252558 ], "independent_source_groups": 47, "brier": 0.4459112569059259, "nll": 0.7654669911888268, "ece10": 0.026559757598291545, "calibration_n": 1173, "confident_wrong": 7, "macro_f1": 0.648782989087528, "confusion_matrix": { "negative": { "negative": 118, "neutral": 41, "positive": 23, "__failed__": 0 }, "neutral": { "negative": 91, "neutral": 363, "positive": 136, "__failed__": 0 }, "positive": { "negative": 40, "neutral": 55, "positive": 306, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "decision2-eos-08b": { "label": "Decision 2.0 Eos 0.8B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 165, "accuracy": 0.825, "wilson95": [ 0.7663556885175099, 0.8713948493372666 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 165, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.825, "brier": 0.2984732049284324, "nll": 0.6263883785145241, "ece10": 0.08007944954520697, "calibration_n": 200, "confident_wrong": 10, "macro_f1": 0.826392717083432, "confusion_matrix": { "world": { "world": 46, "sports": 0, "business": 0, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 11, "sports": 0, "business": 38, "sci_tech": 2, "__failed__": 0 }, "sci_tech": { "world": 6, "sports": 1, "business": 10, "sci_tech": 35, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 102, "accuracy": 0.51, "wilson95": [ 0.44118587678458643, 0.5784372166878682 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 102, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.51, "brier": 0.6862455444425939, "nll": 1.4073981104781792, "ece10": 0.1351521154929428, "calibration_n": 200, "confident_wrong": 6, "macro_f1": 0.42346393140648875, "confusion_matrix": { "sadness": { "sadness": 25, "joy": 4, "love": 2, "anger": 15, "fear": 7, "surprise": 4, "__failed__": 0 }, "joy": { "sadness": 15, "joy": 42, "love": 6, "anger": 8, "fear": 2, "surprise": 6, "__failed__": 0 }, "love": { "sadness": 3, "joy": 4, "love": 3, "anger": 0, "fear": 1, "surprise": 1, "__failed__": 0 }, "anger": { "sadness": 6, "joy": 2, "love": 1, "anger": 18, "fear": 1, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 1, "joy": 2, "love": 1, "anger": 2, "fear": 13, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 1, "joy": 2, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 139, "accuracy": 0.695, "wilson95": [ 0.6280144790202163, 0.7546358436926496 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 139, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.695, "brier": 0.448743850591924, "nll": 0.7026940096727036, "ece10": 0.17760066929130713, "calibration_n": 200, "confident_wrong": 19, "macro_f1": 0.6922223063144882, "confusion_matrix": { "false": { "false": 60, "true": 49, "__failed__": 0 }, "true": { "false": 12, "true": 79, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 152, "accuracy": 0.76, "wilson95": [ 0.6962665902079583, 0.8139338400758631 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 152, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.3245369617838577, "nll": 0.5339609761938623, "ece10": 0.0525983687093, "calibration_n": 200, "confident_wrong": 9, "macro_f1": 0.7558742752517547, "confusion_matrix": { "true": { "true": 63, "false": 14, "__failed__": 0 }, "false": { "true": 34, "false": 89, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 6, "accuracy": 0.5, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.5864467682295508, "nll": 0.9126063059502169, "ece10": 0.35210378695254746, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.4833333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 5, "accuracy": 0.4166666666666667, "wilson95": [ 0.19326031365875665, 0.6804886874504503 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4166666666666667, "brier": 0.8714357142818828, "nll": 1.6447923676156924, "ece10": 0.39604546993829554, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.37333333333333335, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 1, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 2, "documents": 1, "it": 0, "review": 0, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.1279218060598574, "nll": 0.19356118050723511, "ece10": 0.13870694442728582, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.830952380952381, "confusion_matrix": { "design": { "design": 2, "inspection": 0, "purchase": 1, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 2, "work": 1, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.18602537149783735, "nll": 0.29874116512946114, "ece10": 0.15257074927461867, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8035714285714286, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 1, "work": 1, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 73, "accuracy": 0.73, "wilson95": [ 0.6356788323205499, 0.8073041585042365 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 73, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.73, "brier": 0.39259620589630373, "nll": 0.7751591026589614, "ece10": 0.10455175485564425, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.680044275774826, "confusion_matrix": { "label_000": { "label_000": 19, "label_001": 1, "label_002": 0, "label_003": 5, "__failed__": 0 }, "label_001": { "label_000": 13, "label_001": 4, "label_002": 1, "label_003": 7, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 49, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.6857648873726389, "nll": 1.5610186627514644, "ece10": 0.2596564262057102, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.48719993723224436, "confusion_matrix": { "label_000": { "label_000": 10, "label_001": 5, "label_002": 0, "label_003": 0, "label_004": 1, "label_005": 1, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 11, "label_002": 1, "label_003": 0, "label_004": 4, "label_005": 0, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 7, "label_003": 3, "label_004": 2, "label_005": 4, "__failed__": 0 }, "label_003": { "label_000": 2, "label_001": 0, "label_002": 4, "label_003": 7, "label_004": 3, "label_005": 1, "__failed__": 0 }, "label_004": { "label_000": 2, "label_001": 5, "label_002": 2, "label_003": 2, "label_004": 5, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 2, "label_001": 1, "label_002": 3, "label_003": 0, "label_004": 1, "label_005": 9, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 90, "accuracy": 0.9, "wilson95": [ 0.8256343384950865, 0.9447708629393249 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 90, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.16211291910225345, "nll": 0.40851634855276914, "ece10": 0.07063462327294891, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8944444444444445, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 2, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 2, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 2, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 1, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 1, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 1, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0005577463163791157, "nll": 0.005890532791212454, "ece10": 0.0056676717598520415, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 51, "accuracy": 0.4594594594594595, "wilson95": [ 0.36965969632385665, 0.5519713932458781 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 51, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4594594594594595, "brier": 0.6373352267960877, "nll": 1.1250540642357965, "ece10": 0.1706762742760359, "calibration_n": 111, "confident_wrong": 4 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 64, "accuracy": 0.8888888888888888, "wilson95": [ 0.7957706176633595, 0.9426118036505066 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 64, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "cluster_bootstrap95": [ 0.8055555555555556, 0.9583333333333334 ], "independent_source_groups": 36, "brier": 0.16948442962655, "nll": 0.32261952867197113, "ece10": 0.036426577242525994, "calibration_n": 72, "confident_wrong": 1 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 16, "accuracy": 0.8421052631578947, "wilson95": [ 0.6243452472967093, 0.9447952836146585 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8421052631578947, "brier": 0.24497487953574168, "nll": 0.39881812685361473, "ece10": 0.10798022553791395, "calibration_n": 19, "confident_wrong": 1, "macro_f1": 0.8421052631578947, "confusion_matrix": { "false": { "false": 8, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.17935491183517568, "nll": 0.27326147232064374, "ece10": 0.0356260765023133, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 17, "accuracy": 0.7727272727272727, "wilson95": [ 0.5656004682477829, 0.8987696001475147 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7727272727272727, "brier": 0.36981291926142673, "nll": 0.5980527916972652, "ece10": 0.24589321376390372, "calibration_n": 22, "confident_wrong": 4, "macro_f1": 0.7722567287784678, "confusion_matrix": { "false": { "false": 8, "true": 2, "__failed__": 0 }, "true": { "false": 3, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.14526433988904328, "nll": 0.21067575184638118, "ece10": 0.06866822109840494, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8790584415584416, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 3, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 1, "praise": 0, "pricing_billing": 3, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.30500894243194276, "nll": 0.5419005593534719, "ece10": 0.2192781116686417, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8357142857142857, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 3, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02786934901840674, "nll": 0.08710356011706828, "ece10": 0.07445389173874815, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.33646627763721276, "nll": 0.640435091821093, "ece10": 0.25721954095252847, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8023809523809523, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 3, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 2, "refactor": 1, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 1, "security": 0, "style": 0, "refactor": 0, "nit": 3, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 11, "accuracy": 0.7333333333333333, "wilson95": [ 0.4804956594401943, 0.8910254667430761 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7333333333333333, "brier": 0.28926752964512203, "nll": 0.4936250686374836, "ece10": 0.15299250491762317, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7150793650793651, "confusion_matrix": { "feature": { "feature": 2, "fix": 0, "refactor": 1, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 1, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 0, "chore": 3, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.37249143721364103, "nll": 0.8902499745076772, "ece10": 0.22246712313449266, "calibration_n": 17, "confident_wrong": 2, "macro_f1": 0.8235294117647058, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.362243183208376, "nll": 0.5662333696645777, "ece10": 0.16170292288953156, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.75, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 2, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 11, "accuracy": 0.6111111111111112, "wilson95": [ 0.3861904160223791, 0.7969475342783638 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6111111111111112, "brier": 0.5285890442363953, "nll": 1.0119946659240608, "ece10": 0.14362981617779869, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5422222222222223, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 1, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 1, "threat": 0, "self_harm": 1, "spam": 2, "none": 2, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.22382110861165813, "nll": 0.42906533515766365, "ece10": 0.1752832902481724, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8533333333333333, "confusion_matrix": { "news_report": { "news_report": 4, "opinion": 0, "press_release": 0, "analysis": 1, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 1, "opinion": 0, "press_release": 0, "analysis": 1, "feature": 2, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.08060927459805753, "nll": 0.11755127550207978, "ece10": 0.05217658682215355, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 1, "confidentiality": 0, "termination": 0, "indemnification": 3, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 16, "accuracy": 0.8421052631578947, "wilson95": [ 0.6243452472967093, 0.9447952836146585 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8421052631578947, "brier": 0.23687433350885448, "nll": 0.4059437944941554, "ece10": 0.1058398273278769, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8110245310245311, "confusion_matrix": { "address_issue": { "address_issue": 1, "customs_hold": 1, "weather_delay": 0, "failed_attempt": 1, "damaged": 1, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.513754925800014, "nll": 0.8022326967539164, "ece10": 0.20742712772825844, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.5641025641025641, "confusion_matrix": { "false": { "false": 7, "true": 2, "__failed__": 0 }, "true": { "false": 5, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.08712834487314777, "nll": 0.17537092661164522, "ece10": 0.10962214552353117, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9314285714285715, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "purchase_order": 0, "receipt": 2, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.21983171573905733, "nll": 0.4726224311585778, "ece10": 0.1305454088280314, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8428571428571429, "confusion_matrix": { "meeting_request": { "meeting_request": 1, "question": 1, "complaint": 0, "thanks": 1, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.05414289708417354, "nll": 0.1388350570190433, "ece10": 0.1164604686725757, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 3, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.22325042168874104, "nll": 0.3908523858285711, "ece10": 0.12448444829352151, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.8875, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6021089387793948, "nll": 0.8981851211831489, "ece10": 0.2266196291429322, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.49269480519480524, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.4019994117775603, "nll": 0.6960796891794998, "ece10": 0.13952639117900562, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6638655462184874, "confusion_matrix": { "0": { "0": 1, "1": 4, "2": 0, "__failed__": 0 }, "1": { "0": 1, "1": 6, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 14, "accuracy": 0.7, "wilson95": [ 0.4810271816464765, 0.8545227551323956 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.381544394915432, "nll": 0.7337403879697939, "ece10": 0.24060310820760522, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.6802164502164502, "confusion_matrix": { "cheating": { "cheating": 3, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 1, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 1, "other": 0, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 1, "smurfing": 1, "other": 1, "__failed__": 0 }, "other": { "cheating": 1, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 4, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 6, "accuracy": 0.2857142857142857, "wilson95": [ 0.1381386295212213, 0.49956380716085597 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2857142857142857, "brier": 0.9085074653485276, "nll": 1.8955827158482872, "ece10": 0.22955975859470235, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.16695652173913045, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 0, "punctuation": 0, "none": 2, "__failed__": 0 }, "word_choice": { "spelling": 1, "subject_verb_agreement": 1, "word_choice": 0, "punctuation": 0, "none": 2, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.18676415234663896, "nll": 0.3006181455944605, "ece10": 0.1688395424443258, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.875, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 22, "accuracy": 0.9565217391304348, "wilson95": [ 0.7900884492974114, 0.9922833338565469 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9565217391304348, "brier": 0.06319127804383524, "nll": 0.1300976545201437, "ece10": 0.05414141306924359, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 1, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 3, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 12, "accuracy": 0.8571428571428571, "wilson95": [ 0.6005862050302965, 0.9599060787285351 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.3711438403225011, "nll": 0.6886498983162184, "ece10": 0.28236793519604425, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.6761904761904762, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 1, "4": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.36804793393056484, "nll": 0.6886085241969717, "ece10": 0.10122239365335182, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.716540404040404, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 2, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.5282462672409389, "nll": 0.9841105263917543, "ece10": 0.241893795621419, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5913419913419914, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 1, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 1, "1": 1, "2": 1, "3": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.4362696882928409, "nll": 0.6175010043039512, "ece10": 0.17243229973603566, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6458333333333333, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.10284610476329231, "nll": 0.27541424752098215, "ece10": 0.06322112729597701, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9377777777777778, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "sports": 2, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.10749441587972755, "nll": 0.1931732626318449, "ece10": 0.06995247434610656, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "payments": { "payments": 3, "web": 0, "infra": 1, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.2917642768766082, "nll": 0.42685546714348965, "ece10": 0.13177399155752245, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8210526315789474, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3351113430737207, "nll": 0.4909488575637482, "ece10": 0.13777116008959656, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.746031746031746, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3805676940991601, "nll": 0.7133948809533106, "ece10": 0.1492068420756748, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7527065527065527, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 2, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.18959488558285603, "nll": 0.446750166956227, "ece10": 0.20994853543609443, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8690476190476191, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 1, "indian": 0, "thai": 1, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 1, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.7212529988136174, "nll": 1.365489604034509, "ece10": 0.3821146594164355, "calibration_n": 17, "confident_wrong": 5, "macro_f1": 0.4711111111111111, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.2077592041913495, "nll": 0.3869365397000375, "ece10": 0.11543715019393949, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8433333333333334, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 0, "changed_mind": 1, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 0, "changed_mind": 3, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 7, "accuracy": 0.3888888888888889, "wilson95": [ 0.2030524657216362, 0.613809583977621 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3888888888888889, "brier": 0.7174734435211919, "nll": 1.6801408873056178, "ece10": 0.2525904386100211, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.3829004329004329, "confusion_matrix": { "0": { "0": 1, "1": 2, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 1, "3": 0, "4": 1, "__failed__": 0 }, "2": { "0": 1, "1": 1, "2": 1, "3": 0, "4": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 1, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.6196670302302016, "nll": 1.0833602460965839, "ece10": 0.3523758406041273, "calibration_n": 16, "confident_wrong": 4, "macro_f1": 0.6536796536796536, "confusion_matrix": { "false": { "false": 8, "true": 2, "__failed__": 0 }, "true": { "false": 3, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 7, "accuracy": 0.4666666666666667, "wilson95": [ 0.24809535365737387, 0.698830199745016 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.5040596786378712, "nll": 0.6888970120553156, "ece10": 0.269425041259226, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.4444444444444444, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 5, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.3230274105188742, "nll": 0.5546314144828078, "ece10": 0.14442045258198197, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8084848484848486, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 1, "tech": 0, "sales": 2, "account": 0, "other": 1, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.26495110693565876, "nll": 0.41515118573592974, "ece10": 0.22089319130437843, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7571428571428571, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 12, "accuracy": 0.631578947368421, "wilson95": [ 0.41039533164803066, 0.8085048725486492 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.631578947368421, "brier": 0.45864300640240663, "nll": 0.8078340537893491, "ece10": 0.27462945159497554, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.565359477124183, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 4, "1": 0, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.6175482220121797, "nll": 0.8750051367861272, "ece10": 0.25901700237403424, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.5641025641025641, "confusion_matrix": { "false": { "false": 7, "true": 4, "__failed__": 0 }, "true": { "false": 3, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.07643912722961801, "nll": 0.12203921976784057, "ece10": 0.09058353887131017, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8888888888888888, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 13, "accuracy": 0.65, "wilson95": [ 0.43285427668523624, 0.818808175898918 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.5286754402243898, "nll": 0.9267117073152054, "ece10": 0.24544652274529336, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.5861111111111111, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 1, "1": 0, "2": 2, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 20, "accuracy": 1.0, "wilson95": [ 0.8388748419471806, 1.0 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02278774314661968, "nll": 0.06461573796910242, "ece10": 0.05445762890656302, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 12, "accuracy": 0.6, "wilson95": [ 0.3865815007622531, 0.781193467627183 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.4869131269851864, "nll": 0.6873509262178483, "ece10": 0.1933567685856266, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.5833333333333333, "confusion_matrix": { "false": { "false": 8, "true": 1, "__failed__": 0 }, "true": { "false": 7, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.3288238635745756, "nll": 0.6015541068407285, "ece10": 0.3176024517314442, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9080419580419581, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 438, "accuracy": 0.4866666666666667, "wilson95": [ 0.4541383032626595, 0.5193083673168086 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 568, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5293569431500466, "cluster_bootstrap95": [ 0.45513456224304105, 0.5157453124383216 ], "independent_source_groups": 596, "brier": 0.6083350895122563, "nll": 1.1005154955938665, "ece10": 0.21816461647899482, "calibration_n": 1073, "confident_wrong": 62 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 291, "accuracy": 0.5069686411149826, "wilson95": [ 0.4661587786489132, 0.5476858492812584 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 609, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6506410256410257, "brier": 0.49641559562826715, "nll": 0.9777091163091579, "ece10": 0.058783753160003414, "calibration_n": 936, "confident_wrong": 18 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 161, "accuracy": 0.23, "wilson95": [ 0.20034876983953173, 0.26259846740712117 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 362, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3327205882352941, "cluster_bootstrap95": [ 0.1997126436781609, 0.2623194027104924 ], "independent_source_groups": 650, "brier": 0.7873574234739397, "nll": 1.5805255976947226, "ece10": 0.1916340233033244, "calibration_n": 1088, "confident_wrong": 32 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 8, "accuracy": 0.02, "wilson95": [ 0.010168386791664645, 0.03896341576926 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 922, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.461, "brier": 0.710161608511819, "nll": 1.3600161164181792, "ece10": 0.2075431872315776, "calibration_n": 2000, "confident_wrong": 135 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02122825989511389, "nll": 0.06816575406595528, "ece10": 0.06146346881996406, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03853527132258526, "nll": 0.08921488112116975, "ece10": 0.07576193456716394, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 25, "accuracy": 0.8928571428571429, "wilson95": [ 0.7280414845132785, 0.9628816331707 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 25, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8928571428571429, "brier": 0.10892584864610162, "nll": 0.1992740365696188, "ece10": 0.07841131962770385, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.8897435897435899, "confusion_matrix": { "design": { "design": 5, "inspection": 1, "purchase": 0, "work": 1, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 25, "accuracy": 0.8928571428571429, "wilson95": [ 0.7280414845132785, 0.9628816331707 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 25, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8928571428571429, "brier": 0.1465939354342535, "nll": 0.2787860370762386, "ece10": 0.10151831356116434, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.8897435897435897, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 6, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 1, "purchase": 0, "inspection": 1, "design": 5, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 33, "accuracy": 0.825, "wilson95": [ 0.6805000966821229, 0.9125458625396408 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 33, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.825, "brier": 0.25815278935755637, "nll": 0.46931088608568994, "ece10": 0.14327259125309816, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.827310924369748, "confusion_matrix": { "review": { "review": 5, "it": 1, "documents": 0, "procurement": 1, "maintenance": 1, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 2, "it": 0, "documents": 6, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 1, "it": 0, "documents": 0, "procurement": 7, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 1, "it": 0, "documents": 0, "procurement": 0, "maintenance": 7, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 34, "accuracy": 0.85, "wilson95": [ 0.709276756335103, 0.9293881228267964 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.23288394723272293, "nll": 0.4349520625833354, "ece10": 0.10482754674415927, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8492063492063492, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 7, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 6, "it": 0, "review": 2, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 2, "review": 5, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 27, "accuracy": 0.675, "wilson95": [ 0.5201774618987768, 0.799154977682173 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.675, "brier": 0.5213651964865482, "nll": 1.0272066132499382, "ece10": 0.16015791175886462, "calibration_n": 40, "confident_wrong": 1, "macro_f1": 0.6940409590409591, "confusion_matrix": { "review": { "review": 5, "it": 1, "documents": 1, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "it": { "review": 1, "it": 7, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 4, "it": 0, "documents": 4, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 2, "it": 0, "documents": 1, "procurement": 5, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 2, "it": 0, "documents": 0, "procurement": 0, "maintenance": 6, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 26, "accuracy": 0.65, "wilson95": [ 0.49505880837257693, 0.7786547112682372 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 26, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.44277410161668307, "nll": 0.89451385215014, "ece10": 0.1571206200345606, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.6597108155002892, "confusion_matrix": { "maintenance": { "maintenance": 7, "procurement": 0, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 5, "documents": 1, "it": 0, "review": 2, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 4, "it": 0, "review": 4, "__failed__": 0 }, "it": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 6, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 0, "documents": 1, "it": 1, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 211, "accuracy": 0.5275, "wilson95": [ 0.4785460270872868, 0.5759307967243268 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 211, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5275, "brier": 0.6296334373572986, "nll": 1.2711286820244712, "ece10": 0.08474912993234042, "calibration_n": 400, "confident_wrong": 6, "macro_f1": 0.4649925707696621, "confusion_matrix": { "sadness": { "sadness": 59, "joy": 7, "love": 4, "anger": 27, "fear": 16, "surprise": 6, "__failed__": 0 }, "joy": { "sadness": 19, "joy": 62, "love": 14, "anger": 7, "fear": 1, "surprise": 15, "__failed__": 0 }, "love": { "sadness": 7, "joy": 9, "love": 12, "anger": 6, "fear": 1, "surprise": 2, "__failed__": 0 }, "anger": { "sadness": 4, "joy": 4, "love": 3, "anger": 44, "fear": 7, "surprise": 2, "__failed__": 0 }, "fear": { "sadness": 9, "joy": 0, "love": 1, "anger": 7, "fear": 30, "surprise": 5, "__failed__": 0 }, "surprise": { "sadness": 1, "joy": 2, "love": 1, "anger": 1, "fear": 1, "surprise": 4, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 254, "accuracy": 0.8466666666666667, "wilson95": [ 0.8015339429418983, 0.8830335970090831 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 254, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8466666666666667, "brier": 0.21189563897792976, "nll": 0.508138174226567, "ece10": 0.038553318088562834, "calibration_n": 300, "confident_wrong": 5 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 370, "accuracy": 0.925, "wilson95": [ 0.8949522937386957, 0.9469622560771523 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 370, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.12325601266760884, "nll": 0.2179157745137259, "ece10": 0.05792060191897319, "calibration_n": 400, "confident_wrong": 3, "macro_f1": 0.8828125, "confusion_matrix": { "false": { "false": 305, "true": 4, "__failed__": 0 }, "true": { "false": 26, "true": 65, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 316, "accuracy": 0.79, "wilson95": [ 0.7474205944599498, 0.8270622748261584 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 316, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.3187283361899042, "nll": 0.593599432063458, "ece10": 0.10576554114518641, "calibration_n": 400, "confident_wrong": 26, "macro_f1": 0.7740969180674957, "confusion_matrix": { "world": { "world": 87, "sports": 6, "business": 9, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 7, "sports": 115, "business": 0, "sci_tech": 1, "__failed__": 0 }, "business": { "world": 17, "sports": 0, "business": 54, "sci_tech": 1, "__failed__": 0 }, "sci_tech": { "world": 15, "sports": 0, "business": 27, "sci_tech": 60, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 306, "accuracy": 0.765, "wilson95": [ 0.7210494588154521, 0.8039090251873708 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 306, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.765, "brier": 0.34558452051199756, "nll": 0.5471407335887175, "ece10": 0.06989510769675711, "calibration_n": 400, "confident_wrong": 14, "macro_f1": 0.7599530120789602, "confusion_matrix": { "true": { "true": 124, "false": 29, "__failed__": 0 }, "false": { "true": 65, "false": 182, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 230, "accuracy": 0.7666666666666667, "wilson95": [ 0.715618653578636, 0.8109717617682728 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 230, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7666666666666667, "brier": 0.35736411747108915, "nll": 0.9688112389497684, "ece10": 0.056644465304332536, "calibration_n": 300, "confident_wrong": 7 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 300, "accuracy": 0.75, "wilson95": [ 0.7053228670693259, 0.7899209857635259 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 300, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.35668884864610734, "nll": 0.5643277578726522, "ece10": 0.11733714852799002, "calibration_n": 400, "confident_wrong": 26, "macro_f1": 0.7406369955389562, "confusion_matrix": { "false": { "false": 112, "true": 76, "__failed__": 0 }, "true": { "false": 24, "true": 188, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 280, "accuracy": 0.7, "wilson95": [ 0.6533627542224784, 0.7428323280438028 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 280, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.4170768688573033, "nll": 0.6212926675425271, "ece10": 0.06741079435616931, "calibration_n": 400, "confident_wrong": 16, "macro_f1": 0.6808425756004148, "confusion_matrix": { "false": { "false": 189, "true": 11, "__failed__": 0 }, "true": { "false": 109, "true": 91, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 65, "accuracy": 0.65, "wilson95": [ 0.5525444329688796, 0.7363575175690247 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 65, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.5439533591104199, "nll": 0.8683938279638673, "ece10": 0.18946825200048747, "calibration_n": 100, "confident_wrong": 15, "macro_f1": 0.5792763553311696, "confusion_matrix": { "yes": { "yes": 53, "no": 1, "__failed__": 0 }, "no": { "yes": 34, "no": 12, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 54, "accuracy": 0.54, "wilson95": [ 0.44264860323368227, 0.6343919169097589 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 54, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.54, "brier": 0.6458578177787117, "nll": 1.564567340642114, "ece10": 0.14848328768393543, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.5458400458078662, "confusion_matrix": { "balance": { "balance": 3, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 2, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 0, "loan_payment": 0, "mortgage": 2, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 1, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 6, "wire": 1, "statement": 0, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 2, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 }, "statement": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 2, "statement": 3, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 6, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 6, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 2, "branch_appointment": 0, "fee_refund": 7, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 3, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 3, "limit_increase": 0, "other": 1, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 3, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 2, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 3, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 44, "accuracy": 0.44, "wilson95": [ 0.34672026630160796, 0.5377189534832302 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 138, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.69, "brier": 0.4037580836277226, "nll": 0.6577268453947684, "ece10": 0.10861779591777691, "calibration_n": 200, "confident_wrong": 8 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 130, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.49875718545775954, "nll": 1.0760492983203145, "ece10": 0.11847986508315161, "calibration_n": 200, "confident_wrong": 11 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 76, "accuracy": 0.76, "wilson95": [ 0.6676766365018553, 0.8330867444305122 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 76, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.33924274791738074, "nll": 0.9737369417501018, "ece10": 0.07743750991002654, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.7018822494441689, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 2, "receipt": 13, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 1, "receipt": 1, "contract": 1, "resume": 0, "bank_statement": 10, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 1, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 6, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 9, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 6, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 4, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 192, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.7264534965197562, "nll": 1.5143699134021966, "ece10": 0.28285061703453807, "calibration_n": 400, "confident_wrong": 52 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 56, "accuracy": 0.56, "wilson95": [ 0.4622810465167699, 0.6532797336983921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 56, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.56, "brier": 0.5902145492008967, "nll": 1.4449960153280133, "ece10": 0.15317063946273318, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.5512688703737296, "confusion_matrix": { "politics": { "politics": 2, "business": 0, "technology": 3, "science": 1, "health": 5, "sports": 0, "entertainment": 0, "world": 1, "culture": 1, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 2, "technology": 7, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 1, "business": 1, "technology": 1, "science": 6, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 1, "health": 9, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "science": 2, "health": 1, "sports": 4, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 1, "sports": 0, "entertainment": 4, "world": 0, "culture": 2, "weather": 1, "__failed__": 0 }, "world": { "politics": 0, "business": 1, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 1, "world": 4, "culture": 0, "weather": 2, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 1, "health": 2, "sports": 0, "entertainment": 0, "world": 1, "culture": 5, "weather": 1, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 6, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 35, "accuracy": 0.35, "wilson95": [ 0.26364248243097527, 0.44745556703112027 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35, "brier": 0.7761301616613978, "nll": 1.8810656750207468, "ece10": 0.1464477301898447, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.3384697809182935, "confusion_matrix": { "biology": { "biology": 3, "medicine": 1, "chemistry": 1, "physics": 0, "computer_science": 2, "mathematics": 0, "earth_science": 0, "engineering": 1, "economics": 0, "other": 1, "__failed__": 0 }, "medicine": { "biology": 0, "medicine": 4, "chemistry": 0, "physics": 0, "computer_science": 2, "mathematics": 2, "earth_science": 1, "engineering": 0, "economics": 0, "other": 1, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 1, "physics": 0, "computer_science": 2, "mathematics": 0, "earth_science": 0, "engineering": 3, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 6, "computer_science": 2, "mathematics": 0, "earth_science": 3, "engineering": 3, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 0, "chemistry": 1, "physics": 0, "computer_science": 8, "mathematics": 1, "earth_science": 0, "engineering": 5, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 1, "physics": 0, "computer_science": 2, "mathematics": 1, "earth_science": 0, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 4, "mathematics": 0, "earth_science": 2, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 2, "earth_science": 0, "engineering": 5, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 1, "engineering": 1, "economics": 5, "other": 0, "__failed__": 0 }, "other": { "biology": 1, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 1, "mathematics": 2, "earth_science": 1, "engineering": 5, "economics": 1, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 4, "accuracy": 0.04, "wilson95": [ 0.01566330399154762, 0.0983707143588792 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 65, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.325, "brier": 0.6142779929117408, "nll": 1.3938171946600786, "ece10": 0.2068324988915618, "calibration_n": 100, "confident_wrong": 9 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 11, "accuracy": 0.11, "wilson95": [ 0.06254196357064568, 0.18631296503080313 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 89, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.445, "brier": 0.4019992178712932, "nll": 0.6481729835012056, "ece10": 0.1365048534780268, "calibration_n": 100, "confident_wrong": 7 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 69, "accuracy": 0.69, "wilson95": [ 0.5937394428051057, 0.7722030278762395 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.69, "brier": 0.410798261711034, "nll": 0.6748204674408336, "ece10": 0.18190625379493866, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.6388749651907547, "confusion_matrix": { "negative": { "negative": 36, "neutral": 0, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 20, "neutral": 7, "positive": 6, "__failed__": 0 }, "positive": { "negative": 3, "neutral": 2, "positive": 26, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 19, "accuracy": 0.19, "wilson95": [ 0.12514751509768735, 0.27778845379064376 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 101, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.505, "brier": 0.2672071058895284, "nll": 0.3962013909839548, "ece10": 0.06662090387089349, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 216, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.72, "brier": 0.3927619216472069, "nll": 0.759451552629018, "ece10": 0.132940848669993, "calibration_n": 300, "confident_wrong": 12 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.5695480008927328, "nll": 1.685811512965197, "ece10": 0.0894188300828615, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5402807859740634, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 3, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 2, "password_reset": 1, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 6, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 3, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 3, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 1, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 1, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 2, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 3, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 1, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 1, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 2, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 5, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 3, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 1, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 39, "accuracy": 0.39, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.39, "brier": 0.8278961013444054, "nll": 2.147668202732001, "ece10": 0.23009233059165515, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.4147955707738317, "confusion_matrix": { "billing": { "billing": 3, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 2, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 3, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 3, "shipping": 0, "returns": 1, "technical_outage": 4, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 3, "returns": 1, "technical_outage": 3, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 1, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 1, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 0, "account_access": 0, "shipping": 1, "returns": 0, "technical_outage": 7, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 1, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 3, "onboarding": 2, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 4, "onboarding": 0, "integrations": 4, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 2, "onboarding": 0, "integrations": 0, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 2, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 2, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 4, "data_privacy": 1, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 2, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "account_access": 0, "shipping": 1, "returns": 0, "technical_outage": 5, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 1, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 131, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43666666666666665, "brier": 0.7573977279418348, "nll": 1.6744883832242645, "ece10": 0.2135844783932972, "calibration_n": 300, "confident_wrong": 23 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 67, "accuracy": 0.67, "wilson95": [ 0.5730534379509052, 0.7543687726587197 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 67, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.67, "brier": 0.47387655730055406, "nll": 1.189251585524434, "ece10": 0.09246153558655723, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.6447507378769238, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "change_flight": { "new_booking": 2, "change_flight": 6, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 2, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 7, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 2, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 4, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 8, "disruption": 1, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 5, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 2, "visa_docs": 1, "loyalty": 3, "disruption": 1, "other": 2, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 82, "accuracy": 0.82, "wilson95": [ 0.7333264161369926, 0.8829977450105364 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 82, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.82, "brier": 0.267154745645119, "nll": 0.4978408149298421, "ece10": 0.05321251777598908, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.7997137451471817, "confusion_matrix": { "business": { "business": 22, "science and technology": 1, "sports": 0, "world": 2, "__failed__": 0 }, "science and technology": { "business": 9, "science and technology": 10, "sports": 1, "world": 5, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 84, "accuracy": 0.84, "wilson95": [ 0.7557973061072976, 0.8990471151119521 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 84, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.84, "brier": 0.1980607771796021, "nll": 0.4482858989633695, "ece10": 0.05348144549442622, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8393518518518519, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 2, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 1, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 2, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 1, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 1, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 49, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.6586626550545032, "nll": 1.4650336430442967, "ece10": 0.24124646916939366, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.4909615497850792, "confusion_matrix": { "anger": { "anger": 11, "fear": 4, "joy": 0, "love": 0, "sadness": 1, "surprise": 1, "__failed__": 0 }, "fear": { "anger": 1, "fear": 10, "joy": 1, "love": 0, "sadness": 5, "surprise": 0, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 7, "love": 3, "sadness": 3, "surprise": 3, "__failed__": 0 }, "love": { "anger": 2, "fear": 0, "joy": 4, "love": 7, "sadness": 3, "surprise": 1, "__failed__": 0 }, "sadness": { "anger": 2, "fear": 5, "joy": 3, "love": 1, "sadness": 5, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 2, "fear": 1, "joy": 3, "love": 0, "sadness": 1, "surprise": 9, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1989, "accuracy": 0.6746947082767978, "wilson95": [ 0.6575652873232078, 0.6913694418514074 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1989, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6746947082767978, "brier": 0.4901519826634882, "nll": 1.2646824868882254, "ece10": 0.14691230397846916, "calibration_n": 2948, "confident_wrong": 236, "macro_f1": 0.6735496983713866, "confusion_matrix": { "alarm": { "alarm": 63, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 4, "email": 0, "general": 0, "iot": 27, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 8, "audio": 39, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 2, "iot": 9, "lists": 1, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 10, "audio": 0, "calendar": 153, "cooking": 0, "datetime": 129, "email": 18, "general": 4, "iot": 3, "lists": 33, "music": 0, "news": 5, "play": 1, "qa": 3, "recommendation": 37, "social": 2, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 66, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 94, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 6, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 4, "email": 231, "general": 8, "iot": 0, "lists": 12, "music": 0, "news": 1, "play": 0, "qa": 2, "recommendation": 2, "social": 5, "takeaway": 0, "transport": 0, "weather": 3, "__failed__": 0 }, "general": { "alarm": 2, "audio": 0, "calendar": 0, "cooking": 10, "datetime": 18, "email": 10, "general": 46, "iot": 7, "lists": 4, "music": 4, "news": 7, "play": 20, "qa": 24, "recommendation": 15, "social": 8, "takeaway": 2, "transport": 5, "weather": 5, "__failed__": 0 }, "iot": { "alarm": 3, "audio": 1, "calendar": 0, "cooking": 15, "datetime": 0, "email": 0, "general": 1, "iot": 186, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 1, "takeaway": 6, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 1, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 3, "email": 1, "general": 0, "iot": 4, "lists": 128, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 1, "general": 7, "iot": 1, "lists": 7, "music": 51, "news": 0, "play": 8, "qa": 0, "recommendation": 1, "social": 2, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 4, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 112, "play": 1, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 1, "weather": 0, "__failed__": 0 }, "play": { "alarm": 1, "audio": 8, "calendar": 0, "cooking": 0, "datetime": 3, "email": 12, "general": 11, "iot": 13, "lists": 7, "music": 61, "news": 2, "play": 261, "qa": 2, "recommendation": 2, "social": 2, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 5, "datetime": 19, "email": 11, "general": 17, "iot": 10, "lists": 2, "music": 5, "news": 19, "play": 4, "qa": 163, "recommendation": 13, "social": 4, "takeaway": 0, "transport": 10, "weather": 4, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 1, "iot": 2, "lists": 0, "music": 0, "news": 8, "play": 3, "qa": 0, "recommendation": 76, "social": 0, "takeaway": 2, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 7, "general": 7, "iot": 2, "lists": 0, "music": 0, "news": 5, "play": 0, "qa": 0, "recommendation": 0, "social": 84, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 7, "email": 1, "general": 2, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 43, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 29, "email": 6, "general": 0, "iot": 3, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 14, "social": 0, "takeaway": 2, "transport": 64, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 5, "email": 2, "general": 3, "iot": 7, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 129, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1551, "accuracy": 0.5261194029850746, "wilson95": [ 0.508072705278896, 0.5440981183064348 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1551, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5261194029850746, "brier": 0.6700835391326042, "nll": 1.7537503707474986, "ece10": 0.19417659918332392, "calibration_n": 2948, "confident_wrong": 208, "macro_f1": 0.5346075226507654, "confusion_matrix": { "alarm": { "alarm": 67, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 9, "email": 0, "general": 3, "iot": 17, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 3, "audio": 31, "calendar": 0, "cooking": 0, "datetime": 4, "email": 0, "general": 8, "iot": 13, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 12, "audio": 0, "calendar": 73, "cooking": 1, "datetime": 170, "email": 53, "general": 13, "iot": 15, "lists": 14, "music": 1, "news": 7, "play": 0, "qa": 4, "recommendation": 25, "social": 6, "takeaway": 1, "transport": 2, "weather": 3, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 49, "datetime": 8, "email": 1, "general": 3, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 5, "recommendation": 0, "social": 1, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 93, "email": 0, "general": 0, "iot": 2, "lists": 0, "music": 0, "news": 2, "play": 0, "qa": 1, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 7, "email": 220, "general": 8, "iot": 1, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 2, "recommendation": 3, "social": 25, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "general": { "alarm": 2, "audio": 1, "calendar": 0, "cooking": 12, "datetime": 18, "email": 12, "general": 54, "iot": 17, "lists": 3, "music": 6, "news": 9, "play": 7, "qa": 14, "recommendation": 12, "social": 9, "takeaway": 4, "transport": 5, "weather": 2, "__failed__": 0 }, "iot": { "alarm": 4, "audio": 0, "calendar": 0, "cooking": 13, "datetime": 1, "email": 2, "general": 15, "iot": 168, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 7, "transport": 1, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 3, "calendar": 0, "cooking": 0, "datetime": 3, "email": 1, "general": 3, "iot": 7, "lists": 111, "music": 0, "news": 1, "play": 4, "qa": 0, "recommendation": 3, "social": 2, "takeaway": 4, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 1, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 4, "iot": 4, "lists": 5, "music": 57, "news": 0, "play": 6, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 4, "email": 6, "general": 1, "iot": 2, "lists": 3, "music": 0, "news": 93, "play": 3, "qa": 2, "recommendation": 5, "social": 2, "takeaway": 1, "transport": 1, "weather": 0, "__failed__": 0 }, "play": { "alarm": 18, "audio": 24, "calendar": 0, "cooking": 4, "datetime": 10, "email": 9, "general": 25, "iot": 37, "lists": 31, "music": 57, "news": 7, "play": 137, "qa": 0, "recommendation": 6, "social": 18, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "qa": { "alarm": 3, "audio": 1, "calendar": 1, "cooking": 5, "datetime": 22, "email": 9, "general": 26, "iot": 29, "lists": 2, "music": 8, "news": 24, "play": 4, "qa": 115, "recommendation": 14, "social": 6, "takeaway": 2, "transport": 10, "weather": 6, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 3, "email": 2, "general": 5, "iot": 3, "lists": 0, "music": 1, "news": 5, "play": 8, "qa": 0, "recommendation": 62, "social": 0, "takeaway": 3, "transport": 2, "weather": 0, "__failed__": 0 }, "social": { "alarm": 3, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 11, "general": 5, "iot": 5, "lists": 1, "music": 0, "news": 4, "play": 0, "qa": 0, "recommendation": 1, "social": 73, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 7, "datetime": 7, "email": 2, "general": 1, "iot": 5, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 32, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 2, "datetime": 27, "email": 3, "general": 1, "iot": 6, "lists": 1, "music": 0, "news": 2, "play": 0, "qa": 3, "recommendation": 13, "social": 1, "takeaway": 4, "transport": 59, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 43, "email": 4, "general": 3, "iot": 24, "lists": 0, "music": 0, "news": 9, "play": 1, "qa": 1, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 6, "weather": 57, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 626, "accuracy": 0.5336743393009378, "wilson95": [ 0.5050624307833703, 0.5620664075218041 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 626, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5336743393009378, "cluster_bootstrap95": [ 0.5021099693894895, 0.5618592963627792 ], "independent_source_groups": 47, "brier": 0.602963985987441, "nll": 1.0624586089167158, "ece10": 0.16360546283881439, "calibration_n": 1173, "confident_wrong": 51, "macro_f1": 0.508786544146597, "confusion_matrix": { "negative": { "negative": 87, "neutral": 42, "positive": 53, "__failed__": 0 }, "neutral": { "negative": 129, "neutral": 249, "positive": 212, "__failed__": 0 }, "positive": { "negative": 42, "neutral": 69, "positive": 290, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "decision2-kai-06b": { "label": "Decision 2.0 Kai 0.6B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 167, "accuracy": 0.835, "wilson95": [ 0.7773421846094045, 0.8800314467178268 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 167, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.835, "brier": 0.23876398953271255, "nll": 0.4539214009905773, "ece10": 0.03423306873028259, "calibration_n": 200, "confident_wrong": 6, "macro_f1": 0.8358293764514856, "confusion_matrix": { "world": { "world": 45, "sports": 0, "business": 1, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 5, "sports": 0, "business": 43, "sci_tech": 3, "__failed__": 0 }, "sci_tech": { "world": 4, "sports": 1, "business": 14, "sci_tech": 33, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 114, "accuracy": 0.57, "wilson95": [ 0.5007048776415268, 0.6366567766656557 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 114, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.57, "brier": 0.6009159997451833, "nll": 1.2051795830903653, "ece10": 0.15121845892655825, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.455123840196304, "confusion_matrix": { "sadness": { "sadness": 29, "joy": 4, "love": 4, "anger": 18, "fear": 1, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 11, "joy": 55, "love": 5, "anger": 4, "fear": 3, "surprise": 1, "__failed__": 0 }, "love": { "sadness": 1, "joy": 6, "love": 4, "anger": 1, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 6, "joy": 4, "love": 0, "anger": 17, "fear": 0, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 4, "joy": 3, "love": 0, "anger": 1, "fear": 8, "surprise": 4, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 109, "accuracy": 0.545, "wilson95": [ 0.47578611372488105, 0.6125178069011651 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 109, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.545, "brier": 0.5101996171268393, "nll": 0.7046400755448973, "ece10": 0.14989966256286957, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.35275080906148865, "confusion_matrix": { "false": { "false": 109, "true": 0, "__failed__": 0 }, "true": { "false": 91, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 137, "accuracy": 0.685, "wilson95": [ 0.6176504391426966, 0.745376790097715 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 137, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.685, "brier": 0.395779425742393, "nll": 0.5801631871983826, "ece10": 0.062413774053039606, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.6424416129856125, "confusion_matrix": { "true": { "true": 34, "false": 43, "__failed__": 0 }, "false": { "true": 20, "false": 103, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 7, "accuracy": 0.5833333333333334, "wilson95": [ 0.31951131254954973, 0.8067396863412435 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5833333333333334, "brier": 0.47550364635634823, "nll": 0.8804068524847736, "ece10": 0.1284240542796109, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.5771428571428572, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 1, "documents": 1, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 2, "accuracy": 0.16666666666666666, "wilson95": [ 0.04696514218385378, 0.4480308622529734 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 2, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.16666666666666666, "brier": 0.8215948007133456, "nll": 1.4991674552176093, "ece10": 0.2861870202220236, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.06666666666666667, "confusion_matrix": { "maintenance": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 1, "review": 1, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 1, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.22392829326097405, "nll": 0.3794354059199405, "ece10": 0.26215566433068593, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9142857142857144, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 2, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.1507313221255285, "nll": 0.31794088887739225, "ece10": 0.24116465184115785, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9142857142857144, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 1, "purchase": 2, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 83, "accuracy": 0.83, "wilson95": [ 0.7445199523239887, 0.8910643388594006 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 83, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.83, "brier": 0.2578299640260083, "nll": 0.4620415834186571, "ece10": 0.05884702243090308, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.8157979764410472, "confusion_matrix": { "label_000": { "label_000": 22, "label_001": 0, "label_002": 0, "label_003": 3, "__failed__": 0 }, "label_001": { "label_000": 12, "label_001": 11, "label_002": 1, "label_003": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 53, "accuracy": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 53, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.53, "brier": 0.6067726884898922, "nll": 1.2287275449250077, "ece10": 0.1060326035694705, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5344801676533546, "confusion_matrix": { "label_000": { "label_000": 11, "label_001": 1, "label_002": 0, "label_003": 0, "label_004": 4, "label_005": 1, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 11, "label_002": 0, "label_003": 0, "label_004": 4, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 9, "label_003": 2, "label_004": 4, "label_005": 2, "__failed__": 0 }, "label_003": { "label_000": 1, "label_001": 0, "label_002": 6, "label_003": 5, "label_004": 5, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 1, "label_001": 2, "label_002": 2, "label_003": 1, "label_004": 10, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 2, "label_001": 0, "label_002": 3, "label_003": 0, "label_004": 4, "label_005": 7, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 81, "accuracy": 0.81, "wilson95": [ 0.7222115462093562, 0.8748524849023126 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 81, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.81, "brier": 0.25735485593227875, "nll": 0.6832770423050046, "ece10": 0.13960721702872478, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8109126984126984, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 1, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 1, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 1, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 1, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 2, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 2, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 2, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 1, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 47, "accuracy": 0.9791666666666666, "wilson95": [ 0.8910078200474814, 0.9963128890224893 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 47, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9791666666666666, "brier": 0.02957143413336927, "nll": 0.08401928613190017, "ece10": 0.051682626078293076, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 44, "accuracy": 0.3963963963963964, "wilson95": [ 0.31033266426367256, 0.4893912313034269 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 44, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3963963963963964, "brier": 0.6674631200144112, "nll": 1.1253978495906194, "ece10": 0.1317633835727342, "calibration_n": 111, "confident_wrong": 0 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 61, "accuracy": 0.8472222222222222, "wilson95": [ 0.7467771063249946, 0.912492912705243 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 61, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8472222222222222, "cluster_bootstrap95": [ 0.7638888888888888, 0.9305555555555556 ], "independent_source_groups": 36, "brier": 0.27189667046935057, "nll": 0.4827353780516985, "ece10": 0.13564787809251122, "calibration_n": 72, "confident_wrong": 1 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 16, "accuracy": 0.8421052631578947, "wilson95": [ 0.6243452472967093, 0.9447952836146585 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8421052631578947, "brier": 0.2948038505704879, "nll": 0.47018010018181117, "ece10": 0.14400548491629833, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8421052631578947, "confusion_matrix": { "false": { "false": 8, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.3433674129416578, "nll": 0.49709038805618977, "ece10": 0.159966121358826, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6536796536796536, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 14, "accuracy": 0.6363636363636364, "wilson95": [ 0.42951708090131013, 0.8026679532963387 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6363636363636364, "brier": 0.43494546964447456, "nll": 0.6195283701103443, "ece10": 0.19094585480264742, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.6071428571428572, "confusion_matrix": { "false": { "false": 10, "true": 0, "__failed__": 0 }, "true": { "false": 8, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.12141660432712081, "nll": 0.2703883002770248, "ece10": 0.1693726441851419, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9415584415584415, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 3, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 11, "accuracy": 0.6111111111111112, "wilson95": [ 0.3861904160223791, 0.7969475342783638 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6111111111111112, "brier": 0.45770540134736304, "nll": 0.8801131035485039, "ece10": 0.17233425793656199, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5766666666666665, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 1, "3": 0, "4": 2, "__failed__": 0 }, "3": { "0": 1, "1": 1, "2": 0, "3": 1, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.09416120435748598, "nll": 0.2274000621871454, "ece10": 0.11637867163242892, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 1, "roads": 3, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.461210576865078, "nll": 0.943916812472943, "ece10": 0.4530943264023826, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8465608465608465, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 1, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 3, "style": 0, "refactor": 0, "nit": 1, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 4, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 10, "accuracy": 0.6666666666666666, "wilson95": [ 0.41713547738519774, 0.8482367556028526 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.36332015092603615, "nll": 0.746663502834463, "ece10": 0.2534524162049422, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.6499999999999999, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 1, "refactor": 1, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 1, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 1, "refactor": 1, "docs": 0, "test": 0, "chore": 2, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.46858990118945903, "nll": 0.6651590981992341, "ece10": 0.23106222365246076, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7058823529411765, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.3389233032225825, "nll": 0.5186371235101147, "ece10": 0.14238394599591844, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8056680161943319, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.5407948316287601, "nll": 1.066994092121193, "ece10": 0.31877212259077614, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.45714285714285713, "confusion_matrix": { "harassment": { "harassment": 5, "threat": 0, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 1, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 2, "none": 2, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 5, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 15, "accuracy": 0.6818181818181818, "wilson95": [ 0.47318597741042745, 0.8363940681864375 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6818181818181818, "brier": 0.39537525443430305, "nll": 0.7635421849263486, "ece10": 0.18658810551321145, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.6653968253968253, "confusion_matrix": { "news_report": { "news_report": 4, "opinion": 0, "press_release": 1, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 1, "opinion": 4, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 2, "opinion": 0, "press_release": 0, "analysis": 2, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 2, "opinion": 0, "press_release": 1, "analysis": 0, "feature": 1, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.08660380541595285, "nll": 0.18855079140086117, "ece10": 0.1043553174342762, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9377777777777778, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 1, "termination": 2, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 17, "accuracy": 0.8947368421052632, "wilson95": [ 0.6860591728889099, 0.9706414397011299 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8947368421052632, "brier": 0.12673713434884576, "nll": 0.274078513837316, "ece10": 0.10731297783982331, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8929292929292929, "confusion_matrix": { "address_issue": { "address_issue": 2, "customs_hold": 1, "weather_delay": 0, "failed_attempt": 0, "damaged": 1, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.49628328538627714, "nll": 0.6899943955889668, "ece10": 0.08399751383230525, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5641025641025641, "confusion_matrix": { "false": { "false": 7, "true": 2, "__failed__": 0 }, "true": { "false": 5, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.11340859171340037, "nll": 0.26643638625737864, "ece10": 0.13599200932780428, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9047619047619048, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 1, "purchase_order": 1, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 12, "accuracy": 0.8, "wilson95": [ 0.5481455128483065, 0.9295245065301844 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.2751320624668788, "nll": 0.5803421659417359, "ece10": 0.18383011747552647, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7761904761904762, "confusion_matrix": { "meeting_request": { "meeting_request": 1, "question": 0, "complaint": 1, "thanks": 1, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 2, "thanks": 0, "fyi": 1, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0980854255336746, "nll": 0.249577518536203, "ece10": 0.2010563425283103, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 3, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.22668877916555225, "nll": 0.38705725312300365, "ece10": 0.20499427210011545, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8328173374613003, "confusion_matrix": { "false": { "false": 8, "true": 1, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 7, "accuracy": 0.4375, "wilson95": [ 0.23098652405492354, 0.6682144360118811 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4375, "brier": 0.65347848769216, "nll": 1.1862830999671712, "ece10": 0.2511702513332975, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.41507936507936505, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 7, "accuracy": 0.3888888888888889, "wilson95": [ 0.2030524657216362, 0.613809583977621 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3888888888888889, "brier": 0.6263552950968758, "nll": 0.9876993619687229, "ece10": 0.214557463690272, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.2691511387163561, "confusion_matrix": { "0": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "1": { "0": 1, "1": 6, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 5, "2": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 14, "accuracy": 0.7, "wilson95": [ 0.4810271816464765, 0.8545227551323956 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.4662992847903178, "nll": 0.9559218214825382, "ece10": 0.22270993217791396, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7162337662337662, "confusion_matrix": { "cheating": { "cheating": 3, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 1, "other": 0, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 2, "other": 1, "__failed__": 0 }, "other": { "cheating": 1, "harassment": 0, "boosting": 0, "smurfing": 1, "other": 3, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 4, "accuracy": 0.19047619047619047, "wilson95": [ 0.07667565052761793, 0.40000564690204926 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.19047619047619047, "brier": 0.8967970269528258, "nll": 1.8275900904830067, "ece10": 0.21028369198428348, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.1131578947368421, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 0, "punctuation": 0, "none": 2, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 0, "punctuation": 0, "none": 3, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 3, "word_choice": 0, "punctuation": 0, "none": 2, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 2, "none": 3, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.433740486916285, "nll": 0.6119120291505408, "ece10": 0.2590796267670391, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06138096424678877, "nll": 0.20977929557988598, "ece10": 0.18156252931211161, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 9, "accuracy": 0.6428571428571429, "wilson95": [ 0.38764423103907397, 0.8365526824644587 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6428571428571429, "brier": 0.6008686958945669, "nll": 1.1308931440839023, "ece10": 0.31612228804476905, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.54, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 1, "3": 1, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 1, "3": 1, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.5092343898670982, "nll": 0.9574266474265684, "ece10": 0.21743424558066016, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6166666666666667, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.5810374565546349, "nll": 1.0681122335961994, "ece10": 0.22152504944758006, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.4681318681318681, "confusion_matrix": { "0": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 1, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 2, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.47003215754822303, "nll": 0.6633434119576854, "ece10": 0.13509160893026792, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6357142857142857, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 5, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.0936813688336501, "nll": 0.22876552206607292, "ece10": 0.13876795870922, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9377777777777778, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "sports": 2, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.08333558618329051, "nll": 0.2528963822036641, "ece10": 0.2141126814981497, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.5988661253761988, "nll": 0.8025248632929144, "ece10": 0.39350968409466425, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.32, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 9, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.25192391116084134, "nll": 0.4184225955443762, "ece10": 0.19482280859568757, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.49271135381716474, "nll": 0.8577728832950511, "ece10": 0.2701573677176105, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7515151515151516, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "__failed__": 0 }, "2": { "0": 1, "1": 1, "2": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.28331616283740363, "nll": 0.6509805196496283, "ece10": 0.2961571213901889, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8916666666666666, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 2, "indian": 0, "thai": 0, "middle_eastern": 1, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 1, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 6, "accuracy": 0.35294117647058826, "wilson95": [ 0.17309720397243744, 0.5869963651478132 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35294117647058826, "brier": 0.7020915007137286, "nll": 0.9418201757838205, "ece10": 0.2616289765512624, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.35294117647058826, "confusion_matrix": { "false": { "false": 3, "true": 5, "__failed__": 0 }, "true": { "false": 6, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.24027808702717823, "nll": 0.44251166094993927, "ece10": 0.07059840212339044, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7942857142857143, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 0, "changed_mind": 1, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 2, "size_or_fit": 0, "changed_mind": 2, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 8, "accuracy": 0.4444444444444444, "wilson95": [ 0.24559518127005633, 0.6628358435795723 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4444444444444444, "brier": 0.7591431010507824, "nll": 1.5207437092838054, "ece10": 0.06309523643977563, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.45777777777777773, "confusion_matrix": { "0": { "0": 1, "1": 1, "2": 1, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 2, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 1, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 2, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 7, "accuracy": 0.4375, "wilson95": [ 0.23098652405492354, 0.6682144360118811 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4375, "brier": 0.4783732409025743, "nll": 0.6517892161402801, "ece10": 0.3884442788577269, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.41700404858299595, "confusion_matrix": { "false": { "false": 5, "true": 5, "__failed__": 0 }, "true": { "false": 4, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 8, "accuracy": 0.5333333333333333, "wilson95": [ 0.30116980025498397, 0.7519046463426261 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5333333333333333, "brier": 0.5105902365833032, "nll": 0.7067388112773945, "ece10": 0.18646987908745805, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.4444444444444444, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.3010777249949795, "nll": 0.5722520957659996, "ece10": 0.11411114329407263, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8076190476190476, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 2, "account": 0, "other": 2, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.3415646102074851, "nll": 0.5220086300127329, "ece10": 0.17419495610642216, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6886446886446886, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 11, "accuracy": 0.5789473684210527, "wilson95": [ 0.3627590353425504, 0.7685810871754576 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5789473684210527, "brier": 0.5768910523452105, "nll": 1.0604628596505685, "ece10": 0.1889064710607596, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.3972222222222222, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 1, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.5318565005002833, "nll": 0.725874481903306, "ece10": 0.19287000706401267, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.4711111111111111, "confusion_matrix": { "false": { "false": 9, "true": 2, "__failed__": 0 }, "true": { "false": 5, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.17898780977756903, "nll": 0.32304530314537494, "ece10": 0.13505289588308642, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8888888888888888, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 9, "accuracy": 0.45, "wilson95": [ 0.2581978582556243, 0.6579146575496577 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.60656720251973, "nll": 1.0934648285808668, "ece10": 0.23509238306561098, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.28869047619047616, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 2, "1": 0, "2": 1, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.12313093866983935, "nll": 0.23852407720095994, "ece10": 0.10835815859228626, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 8, "accuracy": 0.4, "wilson95": [ 0.21880653237281705, 0.6134184992377469 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.7151210583415654, "nll": 0.9476897695055468, "ece10": 0.3186096054870433, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.2857142857142857, "confusion_matrix": { "false": { "false": 8, "true": 1, "__failed__": 0 }, "true": { "false": 11, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 18, "accuracy": 0.8181818181818182, "wilson95": [ 0.6148339256287345, 0.9269311541657794 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8181818181818182, "brier": 0.5224162635571753, "nll": 0.9717409863351655, "ece10": 0.4486525204822721, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8166666666666668, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 3, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 420, "accuracy": 0.4666666666666667, "wilson95": [ 0.43428397246436723, 0.49933270398430296 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 548, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.51071761416589, "cluster_bootstrap95": [ 0.43909447026690307, 0.4925847417544302 ], "independent_source_groups": 596, "brier": 0.5539142886477157, "nll": 0.9316295091656459, "ece10": 0.09271216140359606, "calibration_n": 1073, "confident_wrong": 2 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 270, "accuracy": 0.47038327526132406, "wilson95": [ 0.4298838019290463, 0.511276529367725 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 581, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6207264957264957, "brier": 0.5106308518990752, "nll": 1.0240385528259397, "ece10": 0.08943680021491339, "calibration_n": 936, "confident_wrong": 0 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 173, "accuracy": 0.24714285714285714, "wilson95": [ 0.21662605794850232, 0.2804197674094742 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 383, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35202205882352944, "cluster_bootstrap95": [ 0.21520265676905875, 0.2804049464291228 ], "independent_source_groups": 650, "brier": 0.745066618048378, "nll": 1.4184735887788933, "ece10": 0.09054570032481649, "calibration_n": 1088, "confident_wrong": 0 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 15, "accuracy": 0.0375, "wilson95": [ 0.022855029814710867, 0.060943842444513396 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 955, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4775, "brier": 0.6168456580497913, "nll": 1.0891837652108416, "ece10": 0.061353307895932754, "calibration_n": 2000, "confident_wrong": 0 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06380799107572212, "nll": 0.16784145380486343, "ece10": 0.14128307420775632, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0726062278474047, "nll": 0.18399857590170182, "ece10": 0.15364168812653814, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "wilson95": [ 0.7735463743160095, 0.9801879358846942 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 26, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.15032549297794268, "nll": 0.31796674929899726, "ece10": 0.1336028324279874, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9283882783882784, "confusion_matrix": { "design": { "design": 6, "inspection": 1, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.15034880975738166, "nll": 0.34694139414283537, "ece10": 0.18117743167152622, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 1, "design": 6, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.25180047609277456, "nll": 0.5133380379352313, "ece10": 0.21250871140804717, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8705368814192344, "confusion_matrix": { "review": { "review": 5, "it": 1, "documents": 0, "procurement": 1, "maintenance": 1, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 7, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "procurement": { "review": 1, "it": 0, "documents": 0, "procurement": 7, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.2320485160391764, "nll": 0.48879824189601645, "ece10": 0.2564489347600351, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8721691672620466, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 1, "procurement": 7, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 7, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 1, "documents": 0, "it": 1, "review": 5, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 31, "accuracy": 0.775, "wilson95": [ 0.6249690325768998, 0.8768390867645927 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 31, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.775, "brier": 0.40081994073544813, "nll": 0.8440738131466254, "ece10": 0.19300226398447437, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7761904761904762, "confusion_matrix": { "review": { "review": 6, "it": 0, "documents": 0, "procurement": 1, "maintenance": 1, "__failed__": 0 }, "it": { "review": 0, "it": 7, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 1, "it": 0, "documents": 7, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 2, "it": 0, "documents": 0, "procurement": 5, "maintenance": 1, "__failed__": 0 }, "maintenance": { "review": 1, "it": 1, "documents": 0, "procurement": 0, "maintenance": 6, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 31, "accuracy": 0.775, "wilson95": [ 0.6249690325768998, 0.8768390867645927 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 31, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.775, "brier": 0.40126556110933886, "nll": 0.8313498891752269, "ece10": 0.19663876285841908, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7758963585434173, "confusion_matrix": { "maintenance": { "maintenance": 6, "procurement": 0, "documents": 0, "it": 1, "review": 1, "__failed__": 0 }, "procurement": { "maintenance": 1, "procurement": 5, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 7, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 7, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 1, "review": 6, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 220, "accuracy": 0.55, "wilson95": [ 0.5010009985806122, 0.5980477719859582 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 220, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.579229620944076, "nll": 1.1804255339218346, "ece10": 0.060506538070317445, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.4576922128387246, "confusion_matrix": { "sadness": { "sadness": 75, "joy": 12, "love": 5, "anger": 19, "fear": 5, "surprise": 3, "__failed__": 0 }, "joy": { "sadness": 11, "joy": 76, "love": 14, "anger": 10, "fear": 1, "surprise": 6, "__failed__": 0 }, "love": { "sadness": 6, "joy": 15, "love": 8, "anger": 7, "fear": 1, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 12, "joy": 5, "love": 3, "anger": 37, "fear": 1, "surprise": 6, "__failed__": 0 }, "fear": { "sadness": 15, "joy": 1, "love": 1, "anger": 8, "fear": 20, "surprise": 7, "__failed__": 0 }, "surprise": { "sadness": 1, "joy": 2, "love": 1, "anger": 1, "fear": 1, "surprise": 4, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 257, "accuracy": 0.8566666666666667, "wilson95": [ 0.8124994228503402, 0.8918152576761502 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 257, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8566666666666667, "brier": 0.21547097878814853, "nll": 0.501523767976839, "ece10": 0.07006341466871163, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 350, "accuracy": 0.875, "wilson95": [ 0.8389809024860414, 0.9038848767632363 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 350, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.20546767777123315, "nll": 0.3549942618482557, "ece10": 0.1065333229555583, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.775895298283358, "confusion_matrix": { "false": { "false": 308, "true": 1, "__failed__": 0 }, "true": { "false": 49, "true": 42, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 335, "accuracy": 0.8375, "wilson95": [ 0.7981666097704109, 0.870412591553939 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 335, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8375, "brier": 0.25764193380294453, "nll": 0.48020646340500567, "ece10": 0.047104300392394854, "calibration_n": 400, "confident_wrong": 16, "macro_f1": 0.8224575610235569, "confusion_matrix": { "world": { "world": 87, "sports": 7, "business": 8, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 2, "sports": 121, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 3, "sports": 1, "business": 66, "sci_tech": 2, "__failed__": 0 }, "sci_tech": { "world": 8, "sports": 0, "business": 33, "sci_tech": 61, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 286, "accuracy": 0.715, "wilson95": [ 0.6688804833361408, 0.7570292301001116 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 286, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.715, "brier": 0.3904870271196408, "nll": 0.5759836785803123, "ece10": 0.055438765914749916, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.6697949252693778, "confusion_matrix": { "true": { "true": 69, "false": 84, "__failed__": 0 }, "false": { "true": 30, "false": 217, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 199, "accuracy": 0.6633333333333333, "wilson95": [ 0.6080919973085686, 0.7144446320914131 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 199, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6633333333333333, "brier": 0.4308637769195354, "nll": 1.059094842296726, "ece10": 0.06418397647225473, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 192, "accuracy": 0.48, "wilson95": [ 0.43146340667729527, 0.5289170850960765 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 192, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.5387235237127875, "nll": 0.7329551169916938, "ece10": 0.20783328753900704, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.34043632673769664, "confusion_matrix": { "false": { "false": 188, "true": 0, "__failed__": 0 }, "true": { "false": 208, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 263, "accuracy": 0.6575, "wilson95": [ 0.6096946457889129, 0.7023089814957836 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 263, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6575, "brier": 0.40535912675085034, "nll": 0.5886470434449453, "ece10": 0.08605053881177062, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.6277957767046343, "confusion_matrix": { "false": { "false": 188, "true": 12, "__failed__": 0 }, "true": { "false": 125, "true": 75, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.48261393914135814, "nll": 0.675714333237283, "ece10": 0.025317954555911126, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5798319327731092, "confusion_matrix": { "yes": { "yes": 30, "no": 24, "__failed__": 0 }, "no": { "yes": 18, "no": 28, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 55, "accuracy": 0.55, "wilson95": [ 0.45244602997442135, 0.6438546202048803 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 55, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.5841443020443856, "nll": 1.3696624442503884, "ece10": 0.08571585194879584, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5575070723909733, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 0, "loan_payment": 1, "mortgage": 2, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 3, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 7, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 2, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 5, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 4, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 6, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 1, "branch_appointment": 1, "fee_refund": 6, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "direct_deposit": { "balance": 1, "transfer": 0, "card_lost": 1, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 3, "limit_increase": 0, "other": 1, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 1, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 142, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.71, "brier": 0.38631272989323356, "nll": 0.6555123056092043, "ece10": 0.10177851149843091, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 127, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.635, "brier": 0.49505592715748564, "nll": 0.9727427970326923, "ece10": 0.11902855094601517, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.3325461174603886, "nll": 0.8485459680681838, "ece10": 0.10201729325744932, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.748098460757629, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 14, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 12, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 1, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 7, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 2, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 9, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 2, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 6, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 186, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.465, "brier": 0.668042998120885, "nll": 1.2209887500849053, "ece10": 0.15512189403789173, "calibration_n": 400, "confident_wrong": 3 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 57, "accuracy": 0.57, "wilson95": [ 0.47215389549212305, 0.6626670147588988 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 57, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.57, "brier": 0.5870684300051682, "nll": 1.2941975785548283, "ece10": 0.1044721076838205, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5641940106826363, "confusion_matrix": { "politics": { "politics": 2, "business": 0, "technology": 3, "science": 1, "health": 5, "sports": 0, "entertainment": 0, "world": 1, "culture": 1, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 8, "technology": 1, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 1, "technology": 8, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 1, "business": 1, "technology": 2, "science": 5, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 1, "sports": 5, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 1, "technology": 0, "science": 1, "health": 1, "sports": 0, "entertainment": 4, "world": 0, "culture": 1, "weather": 1, "__failed__": 0 }, "world": { "politics": 1, "business": 1, "technology": 1, "science": 0, "health": 3, "sports": 0, "entertainment": 1, "world": 2, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 2, "sports": 0, "entertainment": 0, "world": 1, "culture": 7, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 6, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 42, "accuracy": 0.42, "wilson95": [ 0.3279838267435473, 0.5179351329695703 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 42, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42, "brier": 0.761071458148238, "nll": 1.7933489425570637, "ece10": 0.12871235073163192, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.39483161080524165, "confusion_matrix": { "biology": { "biology": 8, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 6, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 2, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 2, "medicine": 0, "chemistry": 1, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 0, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 6, "medicine": 1, "chemistry": 0, "physics": 6, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 2, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 8, "mathematics": 0, "earth_science": 0, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 1, "physics": 0, "computer_science": 2, "mathematics": 2, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 3, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 1, "earth_science": 3, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 2, "earth_science": 0, "engineering": 3, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 1, "engineering": 1, "economics": 5, "other": 0, "__failed__": 0 }, "other": { "biology": 1, "medicine": 0, "chemistry": 0, "physics": 2, "computer_science": 1, "mathematics": 3, "earth_science": 1, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 4, "accuracy": 0.04, "wilson95": [ 0.01566330399154762, 0.0983707143588792 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 72, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.36, "brier": 0.5391208850789709, "nll": 1.0238306307671747, "ece10": 0.09227842871504893, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 13, "accuracy": 0.13, "wilson95": [ 0.07757167427240512, 0.20980351440076428 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 81, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.405, "brier": 0.494524471894728, "nll": 0.8127703887490294, "ece10": 0.07805473783678964, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 68, "accuracy": 0.68, "wilson95": [ 0.5833738138954762, 0.7633085267500092 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 68, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.68, "brier": 0.40357124542077455, "nll": 0.7090409196202081, "ece10": 0.1951895664150172, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6280232039268183, "confusion_matrix": { "negative": { "negative": 33, "neutral": 2, "positive": 1, "__failed__": 0 }, "neutral": { "negative": 14, "neutral": 7, "positive": 12, "__failed__": 0 }, "positive": { "negative": 0, "neutral": 3, "positive": 28, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 15, "accuracy": 0.15, "wilson95": [ 0.0930598528365663, 0.23283559590832364 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 91, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.455, "brier": 0.31298145999484583, "nll": 0.4763509966697479, "ece10": 0.06163699502582122, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 40, "accuracy": 0.4, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 220, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7333333333333333, "brier": 0.40365404116059606, "nll": 0.7428125579539061, "ece10": 0.10876387428027993, "calibration_n": 300, "confident_wrong": 1 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 60, "accuracy": 0.6, "wilson95": [ 0.5020025867910618, 0.6905987135675411 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 60, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5866908369214553, "nll": 1.5972440428343333, "ece10": 0.21280714985920637, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.580539138732416, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 3, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 1, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 4, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 1, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 1, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 3, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 3, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 1, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 2, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 2, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 4, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 3, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 2, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 2, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 40, "accuracy": 0.4, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.7340497893310456, "nll": 1.7073002608872954, "ece10": 0.10355152367087019, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.41572596649996035, "confusion_matrix": { "billing": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 3, "__failed__": 0 }, "account_access": { "billing": 0, "account_access": 4, "shipping": 3, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 1, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 4, "returns": 1, "technical_outage": 1, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 1, "pricing": 0, "data_privacy": 0, "other": 1, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 1, "account_access": 0, "shipping": 1, "returns": 0, "technical_outage": 3, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 3, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 1, "onboarding": 2, "integrations": 3, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 1, "onboarding": 0, "integrations": 4, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 3, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 3, "pricing": 0, "data_privacy": 1, "other": 2, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 3, "data_privacy": 1, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 3, "other": 1, "__failed__": 0 }, "other": { "billing": 1, "account_access": 0, "shipping": 3, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 2, "pricing": 0, "data_privacy": 1, "other": 1, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 4, "accuracy": 0.04, "wilson95": [ 0.01566330399154762, 0.0983707143588792 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 116, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.38666666666666666, "brier": 0.6802449437085254, "nll": 1.3941227024211031, "ece10": 0.12125852877617002, "calibration_n": 300, "confident_wrong": 0 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 60, "accuracy": 0.6, "wilson95": [ 0.5020025867910618, 0.6905987135675411 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 60, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5603982062526418, "nll": 1.2859410196142247, "ece10": 0.11965015482644087, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5865362862589384, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 5, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 2, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 7, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 7, "car": 0, "visa_docs": 1, "loyalty": 0, "disruption": 2, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 3, "loyalty": 2, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 7, "disruption": 1, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 1, "disruption": 4, "other": 1, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 2, "car": 1, "visa_docs": 1, "loyalty": 3, "disruption": 2, "other": 2, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 82, "accuracy": 0.82, "wilson95": [ 0.7333264161369926, 0.8829977450105364 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 82, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.82, "brier": 0.2348726659165348, "nll": 0.4324616588125602, "ece10": 0.08607047621124105, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.8105873001777394, "confusion_matrix": { "business": { "business": 22, "science and technology": 0, "sports": 0, "world": 3, "__failed__": 0 }, "science and technology": { "business": 11, "science and technology": 12, "sports": 1, "world": 1, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 1, "science and technology": 1, "sports": 0, "world": 23, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 81, "accuracy": 0.81, "wilson95": [ 0.7222115462093562, 0.8748524849023126 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 81, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.81, "brier": 0.2582749820430069, "nll": 0.692943491642301, "ece10": 0.1471329108433259, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8016534391534392, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 1, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 1, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 1, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 2, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 2, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 2, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 53, "accuracy": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 53, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.53, "brier": 0.6315653513399709, "nll": 1.2825976940947152, "ece10": 0.1485330426219207, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5405435251023486, "confusion_matrix": { "anger": { "anger": 12, "fear": 1, "joy": 0, "love": 0, "sadness": 3, "surprise": 1, "__failed__": 0 }, "fear": { "anger": 1, "fear": 8, "joy": 0, "love": 0, "sadness": 7, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 9, "love": 2, "sadness": 4, "surprise": 2, "__failed__": 0 }, "love": { "anger": 1, "fear": 0, "joy": 5, "love": 6, "sadness": 5, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 1, "fear": 2, "joy": 2, "love": 1, "sadness": 9, "surprise": 1, "__failed__": 0 }, "surprise": { "anger": 2, "fear": 0, "joy": 1, "love": 0, "sadness": 4, "surprise": 9, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2149, "accuracy": 0.7289687924016283, "wilson95": [ 0.7126331699840927, 0.7447084653418594 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2149, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7289687924016283, "brier": 0.38216662285406394, "nll": 0.8750897966700363, "ece10": 0.03049999693101547, "calibration_n": 2948, "confident_wrong": 29, "macro_f1": 0.735920658505035, "confusion_matrix": { "alarm": { "alarm": 66, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 9, "email": 0, "general": 0, "iot": 18, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 0, "audio": 40, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 11, "iot": 10, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 11, "audio": 0, "calendar": 275, "cooking": 0, "datetime": 70, "email": 6, "general": 4, "iot": 0, "lists": 8, "music": 0, "news": 2, "play": 0, "qa": 3, "recommendation": 17, "social": 1, "takeaway": 2, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 67, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 97, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 2, "email": 232, "general": 14, "iot": 0, "lists": 8, "music": 0, "news": 1, "play": 0, "qa": 3, "recommendation": 0, "social": 2, "takeaway": 0, "transport": 2, "weather": 2, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 2, "cooking": 4, "datetime": 10, "email": 7, "general": 101, "iot": 6, "lists": 3, "music": 3, "news": 5, "play": 3, "qa": 17, "recommendation": 10, "social": 4, "takeaway": 3, "transport": 5, "weather": 3, "__failed__": 0 }, "iot": { "alarm": 2, "audio": 3, "calendar": 0, "cooking": 12, "datetime": 2, "email": 0, "general": 12, "iot": 182, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 3, "email": 1, "general": 2, "iot": 4, "lists": 127, "music": 2, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 6, "iot": 0, "lists": 5, "music": 65, "news": 0, "play": 3, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 4, "email": 3, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 105, "play": 0, "qa": 4, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "play": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 1, "email": 7, "general": 27, "iot": 19, "lists": 3, "music": 77, "news": 1, "play": 247, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 1, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 19, "email": 4, "general": 65, "iot": 9, "lists": 2, "music": 3, "news": 5, "play": 3, "qa": 142, "recommendation": 7, "social": 4, "takeaway": 1, "transport": 17, "weather": 5, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 6, "iot": 1, "lists": 0, "music": 0, "news": 4, "play": 9, "qa": 1, "recommendation": 66, "social": 0, "takeaway": 0, "transport": 5, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 12, "general": 6, "iot": 1, "lists": 2, "music": 0, "news": 3, "play": 0, "qa": 0, "recommendation": 0, "social": 81, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 3, "email": 0, "general": 0, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 50, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 24, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 7, "social": 0, "takeaway": 3, "transport": 79, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 3, "email": 0, "general": 15, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 2, "weather": 127, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1614, "accuracy": 0.5474898236092266, "wilson95": [ 0.5294721556683573, 0.5653838872140624 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1614, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5474898236092266, "brier": 0.5967794764432004, "nll": 1.4954847722994018, "ece10": 0.039268749412316895, "calibration_n": 2948, "confident_wrong": 19, "macro_f1": 0.5669663267562415, "confusion_matrix": { "alarm": { "alarm": 71, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 11, "email": 0, "general": 2, "iot": 11, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 4, "audio": 31, "calendar": 0, "cooking": 1, "datetime": 3, "email": 0, "general": 13, "iot": 6, "lists": 1, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 7, "audio": 0, "calendar": 176, "cooking": 1, "datetime": 133, "email": 11, "general": 33, "iot": 8, "lists": 12, "music": 1, "news": 3, "play": 0, "qa": 4, "recommendation": 6, "social": 1, "takeaway": 3, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 44, "datetime": 4, "email": 0, "general": 7, "iot": 3, "lists": 1, "music": 0, "news": 0, "play": 1, "qa": 10, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 96, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 7, "email": 226, "general": 21, "iot": 1, "lists": 5, "music": 0, "news": 2, "play": 0, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 2, "weather": 0, "__failed__": 0 }, "general": { "alarm": 1, "audio": 2, "calendar": 3, "cooking": 8, "datetime": 18, "email": 2, "general": 100, "iot": 8, "lists": 1, "music": 3, "news": 4, "play": 3, "qa": 16, "recommendation": 5, "social": 0, "takeaway": 3, "transport": 6, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 17, "calendar": 0, "cooking": 14, "datetime": 6, "email": 1, "general": 34, "iot": 135, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 2, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 2, "calendar": 2, "cooking": 0, "datetime": 2, "email": 0, "general": 16, "iot": 5, "lists": 102, "music": 6, "news": 0, "play": 3, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 3, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 15, "iot": 1, "lists": 1, "music": 58, "news": 0, "play": 3, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 4, "email": 5, "general": 17, "iot": 0, "lists": 1, "music": 0, "news": 88, "play": 0, "qa": 5, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 1, "weather": 1, "__failed__": 0 }, "play": { "alarm": 3, "audio": 19, "calendar": 0, "cooking": 5, "datetime": 5, "email": 10, "general": 86, "iot": 34, "lists": 13, "music": 92, "news": 2, "play": 108, "qa": 1, "recommendation": 1, "social": 2, "takeaway": 2, "transport": 2, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 2, "datetime": 29, "email": 3, "general": 78, "iot": 13, "lists": 1, "music": 2, "news": 2, "play": 4, "qa": 127, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 20, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 1, "calendar": 3, "cooking": 1, "datetime": 5, "email": 0, "general": 12, "iot": 3, "lists": 1, "music": 4, "news": 3, "play": 11, "qa": 1, "recommendation": 32, "social": 0, "takeaway": 4, "transport": 13, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 9, "general": 17, "iot": 5, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 0, "recommendation": 0, "social": 71, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 6, "datetime": 6, "email": 0, "general": 4, "iot": 4, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 35, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 24, "email": 2, "general": 15, "iot": 6, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 4, "social": 0, "takeaway": 8, "transport": 55, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 42, "email": 1, "general": 15, "iot": 10, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 6, "recommendation": 1, "social": 0, "takeaway": 4, "transport": 11, "weather": 59, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 677, "accuracy": 0.577152600170503, "wilson95": [ 0.5486751333091117, 0.605126382301478 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 677, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.577152600170503, "cluster_bootstrap95": [ 0.5515960170335105, 0.6038189143341816 ], "independent_source_groups": 47, "brier": 0.5565252450099746, "nll": 0.9283850307104583, "ece10": 0.06474422289344399, "calibration_n": 1173, "confident_wrong": 0, "macro_f1": 0.543275449679602, "confusion_matrix": { "negative": { "negative": 100, "neutral": 66, "positive": 16, "__failed__": 0 }, "neutral": { "negative": 130, "neutral": 381, "positive": 79, "__failed__": 0 }, "positive": { "negative": 76, "neutral": 129, "positive": 196, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "decision2-nox-4b": { "label": "Decision 2.0 Nox 4B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 174, "accuracy": 0.87, "wilson95": [ 0.8163364812775159, 0.9097179772033067 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 174, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.19891208857638365, "nll": 0.3604046813218759, "ece10": 0.054944460917735104, "calibration_n": 200, "confident_wrong": 4, "macro_f1": 0.8726788168578128, "confusion_matrix": { "world": { "world": 44, "sports": 0, "business": 2, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 47, "sci_tech": 2, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 13, "sci_tech": 37, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 118, "accuracy": 0.59, "wilson95": [ 0.5207645903369208, 0.6558432509151711 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 118, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.59, "brier": 0.6142903946479399, "nll": 1.2265249766351847, "ece10": 0.12635538256318152, "calibration_n": 200, "confident_wrong": 7, "macro_f1": 0.461245944971765, "confusion_matrix": { "sadness": { "sadness": 38, "joy": 3, "love": 2, "anger": 8, "fear": 6, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 15, "joy": 46, "love": 8, "anger": 6, "fear": 2, "surprise": 2, "__failed__": 0 }, "love": { "sadness": 2, "joy": 4, "love": 5, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 }, "anger": { "sadness": 7, "joy": 2, "love": 1, "anger": 17, "fear": 1, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 4, "joy": 1, "love": 1, "anger": 1, "fear": 12, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 186, "accuracy": 0.93, "wilson95": [ 0.8859451911438936, 0.9578478281716571 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 186, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.93, "brier": 0.1209913187821238, "nll": 0.21173336207871568, "ece10": 0.051246475070312644, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.9294283697953423, "confusion_matrix": { "false": { "false": 102, "true": 7, "__failed__": 0 }, "true": { "false": 7, "true": 84, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 185, "accuracy": 0.925, "wilson95": [ 0.8799563897637055, 0.954025082815618 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 185, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.11696930074108831, "nll": 0.21007929085752977, "ece10": 0.060558283396076835, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.9206160196872271, "confusion_matrix": { "true": { "true": 69, "false": 8, "__failed__": 0 }, "false": { "true": 7, "false": 116, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": 0.7, "development": { "n": 87, "correct": 87, "wilson95": [ 0.9577124930558795, 0.9999999999999999 ] }, "test_coverage": 0.895, "accepted_n": 179, "accepted_correct": 172, "accepted_accuracy": 0.9608938547486033, "accepted_wilson95": [ 0.9214910989715486, 0.9809300535626226 ], "errors_per_1000_all_test_cases": 35.0, "note": "Selected on development only, applied unchanged to test. An empirical pilot estimate, not a production guarantee." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.1973404436470384, "nll": 0.3759247654723474, "ece10": 0.13423910085356633, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8533333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.30312594381380104, "nll": 0.6555570377434625, "ece10": 0.21504290432537745, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8533333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.065169349697962, "nll": 0.1264282920807366, "ece10": 0.0978465339283189, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9142857142857143, "confusion_matrix": { "design": { "design": 2, "inspection": 0, "purchase": 1, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.01778003201564767, "nll": 0.06935884841309017, "ece10": 0.06361458369723726, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 85, "accuracy": 0.85, "wilson95": [ 0.7671644040916763, 0.9069401471634337 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 85, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.21375037475498065, "nll": 0.3627664137898304, "ece10": 0.08115424039441346, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.829952700683251, "confusion_matrix": { "label_000": { "label_000": 25, "label_001": 0, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 12, "label_001": 10, "label_002": 1, "label_003": 2, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 51, "accuracy": 0.51, "wilson95": [ 0.4134801193402717, 0.6057800106955886 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 51, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.51, "brier": 0.6377739569814431, "nll": 1.3464532877259883, "ece10": 0.13195013443187742, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.512612734192676, "confusion_matrix": { "label_000": { "label_000": 11, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 3, "label_005": 1, "__failed__": 0 }, "label_001": { "label_000": 2, "label_001": 11, "label_002": 0, "label_003": 0, "label_004": 4, "label_005": 0, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 7, "label_003": 4, "label_004": 3, "label_005": 2, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 3, "label_003": 6, "label_004": 8, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 1, "label_002": 2, "label_003": 2, "label_004": 11, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 2, "label_001": 0, "label_002": 3, "label_003": 0, "label_004": 6, "label_005": 5, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 86, "accuracy": 0.86, "wilson95": [ 0.7786281178903616, 0.9147365634006086 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.23113922068027307, "nll": 0.5021653464675238, "ece10": 0.1101888879499813, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8574074074074073, "confusion_matrix": { "label_000": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 1, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 2, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 2, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 2, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0017567108974285426, "nll": 0.01238864104980196, "ece10": 0.011898500311822303, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 69, "accuracy": 0.6216216216216216, "wilson95": [ 0.5287598467882427, 0.7063468845025537 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6216216216216216, "brier": 0.5630970897883144, "nll": 0.9809851388072315, "ece10": 0.1541995446250885, "calibration_n": 111, "confident_wrong": 5 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 68, "accuracy": 0.9444444444444444, "wilson95": [ 0.8656798403400208, 0.9781857840186834 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 68, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "cluster_bootstrap95": [ 0.8611111111111112, 1.0 ], "independent_source_groups": 36, "brier": 0.06885019241870846, "nll": 0.14552578068065655, "ece10": 0.0680769975132362, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.12168693147292149, "nll": 0.2272645026538754, "ece10": 0.1747747294911423, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9467787114845938, "confusion_matrix": { "false": { "false": 10, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.013809101282094694, "nll": 0.05815319524168652, "ece10": 0.0542197899504306, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.13105781945658912, "nll": 0.22357419219862731, "ece10": 0.08060683895546232, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.905982905982906, "confusion_matrix": { "false": { "false": 8, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.021340932964169352, "nll": 0.07080279084571808, "ece10": 0.06405468101436257, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.22102788873693277, "nll": 0.46081492971165944, "ece10": 0.2869775007640655, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.04062312444493119, "nll": 0.07069946232273058, "ece10": 0.05235285670752193, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 1, "roads": 3, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.20461695166272845, "nll": 0.4133904205749789, "ece10": 0.2513798795818008, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8873015873015873, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 1, "security": 0, "style": 1, "refactor": 0, "nit": 2, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.17395164772855734, "nll": 0.3518959724400725, "ece10": 0.21507199771286994, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8277777777777778, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 1, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.1396242798869085, "nll": 0.22022743056218694, "ece10": 0.1361659527821684, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9403508771929825, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.1891053621399993, "nll": 0.3179593545943325, "ece10": 0.11046920611805379, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.2020556948926259, "nll": 0.3786595512695487, "ece10": 0.15330495555068144, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.74989898989899, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 1, "threat": 1, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 1, "none": 4, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.18532908278484259, "nll": 0.3536748924678206, "ece10": 0.15603125868609158, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 2, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 2, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.009237241763306191, "nll": 0.03989593104612531, "ece10": 0.03707706106666603, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.000971203291442274, "nll": 0.01941397990176632, "ece10": 0.019082306693141416, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.05820960161301801, "nll": 0.1571393178494485, "ece10": 0.13929516569853784, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.002728988269985508, "nll": 0.019865447228905753, "ece10": 0.01897448636590743, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.13033108927909348, "nll": 0.27574878722776097, "ece10": 0.10158536980762756, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.9314285714285713, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.09115113034308861, "nll": 0.14567902590111154, "ece10": 0.09366312114266884, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.12158018659790747, "nll": 0.21408782481150554, "ece10": 0.16514105963810882, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9442724458204335, "confusion_matrix": { "false": { "false": 8, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.45508317854895913, "nll": 0.6994420444637443, "ece10": 0.15876589447517248, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5535714285714286, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.16492914697881633, "nll": 0.3557083920122302, "ece10": 0.240402525509667, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9407407407407407, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.1666659224249557, "nll": 0.3632228752827567, "ece10": 0.21131552469419182, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 4, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 1, "other": 0, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 3, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 12, "accuracy": 0.5714285714285714, "wilson95": [ 0.3654655436930671, 0.7553003107462404 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5714285714285714, "brier": 0.6603880484453496, "nll": 1.267001570710476, "ece10": 0.25587856828916433, "calibration_n": 21, "confident_wrong": 1, "macro_f1": 0.579548872180451, "confusion_matrix": { "spelling": { "spelling": 1, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 3, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 3, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 0, "none": 3, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 2, "none": 3, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.06594001656989615, "nll": 0.1126023736801238, "ece10": 0.07933334790017466, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9352226720647774, "confusion_matrix": { "false": { "false": 6, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.002933303792072669, "nll": 0.0324139086107701, "ece10": 0.031486873846656833, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 12, "accuracy": 0.8571428571428571, "wilson95": [ 0.6005862050302965, 0.9599060787285351 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.29447301556555255, "nll": 0.5906008510052063, "ece10": 0.2741992148824705, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.75, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.2705373945403754, "nll": 0.5164384725476682, "ece10": 0.31063551314060023, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8875, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.37095930850056325, "nll": 0.7044275079601918, "ece10": 0.28862169637275437, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8666666666666667, "confusion_matrix": { "0": { "0": 2, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.14084535230262957, "nll": 0.24805361251699598, "ece10": 0.19047379409110599, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9411764705882353, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02451058864772042, "nll": 0.08276649191337866, "ece10": 0.07293999231630201, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.09803339354452323, "nll": 0.20963483192597845, "ece10": 0.08207333017576829, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "payments": { "payments": 3, "web": 0, "infra": 1, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.29796263220327157, "nll": 0.4098499238244553, "ece10": 0.15231365416611645, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6640316205533596, "confusion_matrix": { "false": { "false": 3, "true": 5, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.01381901679554806, "nll": 0.06848209444386343, "ece10": 0.06467798259634523, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.17809735052901385, "nll": 0.36015314020730366, "ece10": 0.2157517775474598, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9326599326599326, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.07890878815549893, "nll": 0.15861205265862874, "ece10": 0.06960356995735305, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8873015873015873, "confusion_matrix": { "italian": { "italian": 1, "french": 1, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 1, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.18246316175781652, "nll": 0.3238234735065044, "ece10": 0.12838101655080794, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8819444444444444, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.12088737450093887, "nll": 0.22161087567707793, "ece10": 0.13020913938481238, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.4853296073842469, "nll": 1.0323060117640328, "ece10": 0.10452691847626319, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6490476190476191, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 2, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.36940812441322535, "nll": 0.4957260949651589, "ece10": 0.256886739383804, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6862745098039216, "confusion_matrix": { "false": { "false": 5, "true": 5, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 10, "accuracy": 0.6666666666666666, "wilson95": [ 0.41713547738519774, 0.8482367556028526 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.3796657915696006, "nll": 0.5232630282928847, "ece10": 0.3366829772769888, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.6411483253588517, "confusion_matrix": { "false": { "false": 3, "true": 5, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.1115964550419408, "nll": 0.2258707909458522, "ece10": 0.1653129192559536, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8732467532467533, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 1, "tech": 0, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.13706825546450419, "nll": 0.26262748369706074, "ece10": 0.21580366578914814, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 14, "accuracy": 0.7368421052631579, "wilson95": [ 0.5120844910218489, 0.8819358765321748 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7368421052631579, "brier": 0.3365507419729863, "nll": 0.6089346454068664, "ece10": 0.24304391769968017, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.6105769230769231, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 2, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.2210980390203402, "nll": 0.3585898621429372, "ece10": 0.273466594840016, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7571428571428571, "confusion_matrix": { "false": { "false": 8, "true": 3, "__failed__": 0 }, "true": { "false": 1, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04421136078624451, "nll": 0.12229832071698679, "ece10": 0.10817045342413109, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 16, "accuracy": 0.8, "wilson95": [ 0.5839825677481064, 0.919342337420202 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.3603615104412395, "nll": 0.6439468638932127, "ece10": 0.19910457922663702, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8127705627705628, "confusion_matrix": { "0": { "0": 4, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 20, "accuracy": 1.0, "wilson95": [ 0.8388748419471806, 1.0 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.011970862530863375, "nll": 0.03197256593009993, "ece10": 0.02788002294084101, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 17, "accuracy": 0.85, "wilson95": [ 0.6395811352592431, 0.9476312541037835 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.2656983946115504, "nll": 0.4584219152757223, "ece10": 0.1176204322415618, "calibration_n": 20, "confident_wrong": 1, "macro_f1": 0.849624060150376, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.20202904781193345, "nll": 0.43997309264241513, "ece10": 0.1931281074674894, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9142857142857143, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 2, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 588, "accuracy": 0.6533333333333333, "wilson95": [ 0.6216488275174086, 0.683714460818708 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 752, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.700838769804287, "cluster_bootstrap95": [ 0.6137904392174773, 0.6972258381776191 ], "independent_source_groups": 596, "brier": 0.3994641824528917, "nll": 0.713220192674444, "ece10": 0.06367309248538708, "calibration_n": 1073, "confident_wrong": 29 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 403, "accuracy": 0.7020905923344948, "wilson95": [ 0.6634337654552191, 0.738060444519753 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 750, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8012820512820513, "brier": 0.2881777882360119, "nll": 0.5615982002594853, "ece10": 0.04853014125366999, "calibration_n": 936, "confident_wrong": 9 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 352, "accuracy": 0.5028571428571429, "wilson95": [ 0.4659035282504132, 0.5397795697681409 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 672, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6176470588235294, "cluster_bootstrap95": [ 0.4680179533885617, 0.5381414701803051 ], "independent_source_groups": 650, "brier": 0.48633638391221307, "nll": 0.9149375753475447, "ece10": 0.040586299152221186, "calibration_n": 1088, "confident_wrong": 15 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 76, "accuracy": 0.19, "wilson95": [ 0.15457385084236502, 0.2313237716448988 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1360, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.68, "brier": 0.44174886957729886, "nll": 0.7768941293250108, "ece10": 0.06509724675386781, "calibration_n": 2000, "confident_wrong": 9 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.008709071702229102, "nll": 0.04487535892515031, "ece10": 0.04205297518363702, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.027459085021526775, "nll": 0.06662147497397079, "ece10": 0.05556861129508253, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 0, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.016751869651559063, "nll": 0.07128060340818719, "ece10": 0.06513701731431296, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.01784443553567067, "nll": 0.0758593853566915, "ece10": 0.06980432875210965, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.1364178293368382, "nll": 0.2968150471838984, "ece10": 0.11505517163475146, "calibration_n": 40, "confident_wrong": 1, "macro_f1": 0.9498039215686275, "confusion_matrix": { "review": { "review": 7, "it": 0, "documents": 0, "procurement": 1, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 7, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.10381527113289994, "nll": 0.22741405974355042, "ece10": 0.09826164089761916, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9498039215686275, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 7, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 34, "accuracy": 0.85, "wilson95": [ 0.709276756335103, 0.9293881228267964 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.2244684670349657, "nll": 0.4353374882658856, "ece10": 0.08488533530926969, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8415686274509804, "confusion_matrix": { "review": { "review": 4, "it": 0, "documents": 1, "procurement": 1, "maintenance": 2, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 6, "procurement": 0, "maintenance": 2, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.16712662820186625, "nll": 0.3498301318273548, "ece10": 0.11931748225382625, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8625816993464053, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 7, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 1, "it": 2, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 232, "accuracy": 0.58, "wilson95": [ 0.5310957511319983, 0.6273822817745143 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 232, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.5554474386735727, "nll": 1.1253715559078217, "ece10": 0.05388467630804441, "calibration_n": 400, "confident_wrong": 3, "macro_f1": 0.5074747033063747, "confusion_matrix": { "sadness": { "sadness": 85, "joy": 9, "love": 7, "anger": 10, "fear": 7, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 18, "joy": 60, "love": 22, "anger": 8, "fear": 2, "surprise": 8, "__failed__": 0 }, "love": { "sadness": 9, "joy": 9, "love": 11, "anger": 4, "fear": 2, "surprise": 2, "__failed__": 0 }, "anger": { "sadness": 16, "joy": 3, "love": 2, "anger": 39, "fear": 0, "surprise": 4, "__failed__": 0 }, "fear": { "sadness": 9, "joy": 0, "love": 1, "anger": 5, "fear": 33, "surprise": 4, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 0, "love": 1, "anger": 1, "fear": 2, "surprise": 4, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 271, "accuracy": 0.9033333333333333, "wilson95": [ 0.8646182980304887, 0.9318497051817106 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 271, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9033333333333333, "brier": 0.12766746853902305, "nll": 0.32212835148930447, "ece10": 0.036354230289214966, "calibration_n": 300, "confident_wrong": 1 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 315, "accuracy": 0.7875, "wilson95": [ 0.7447739406814162, 0.8247564900763633 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 315, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7875, "brier": 0.26680195891762015, "nll": 0.3941594083862778, "ece10": 0.09179013954320614, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.75108169645145, "confusion_matrix": { "false": { "false": 234, "true": 75, "__failed__": 0 }, "true": { "false": 10, "true": 81, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 342, "accuracy": 0.855, "wilson95": [ 0.8171167606818923, 0.8861295103407572 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 342, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.855, "brier": 0.2271688810124376, "nll": 0.438838940650939, "ece10": 0.0640638220212186, "calibration_n": 400, "confident_wrong": 25, "macro_f1": 0.8420445632798574, "confusion_matrix": { "world": { "world": 88, "sports": 6, "business": 8, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 2, "sports": 121, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 68, "sci_tech": 2, "__failed__": 0 }, "sci_tech": { "world": 9, "sports": 0, "business": 28, "sci_tech": 65, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 368, "accuracy": 0.92, "wilson95": [ 0.8892454231693245, 0.9427642495898665 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 368, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.92, "brier": 0.11192681537907789, "nll": 0.20642461314686525, "ece10": 0.07082327964958209, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.9139576779328332, "confusion_matrix": { "true": { "true": 131, "false": 22, "__failed__": 0 }, "false": { "true": 10, "false": 237, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 225, "accuracy": 0.75, "wilson95": [ 0.6980484035459826, 0.7956301108417443 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 225, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3348382934461596, "nll": 0.7812731987556473, "ece10": 0.09305213357886005, "calibration_n": 300, "confident_wrong": 1 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 374, "accuracy": 0.935, "wilson95": [ 0.9064647858253625, 0.9552595181037997 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 374, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.935, "brier": 0.10075860172742296, "nll": 0.18484014743715857, "ece10": 0.038067201053525554, "calibration_n": 400, "confident_wrong": 3, "macro_f1": 0.9348027784046742, "confusion_matrix": { "false": { "false": 176, "true": 12, "__failed__": 0 }, "true": { "false": 14, "true": 198, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 336, "accuracy": 0.84, "wilson95": [ 0.8008643954521164, 0.8726672444005619 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 336, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.84, "brier": 0.24423253509863257, "nll": 0.3881451006209936, "ece10": 0.07026529469297274, "calibration_n": 400, "confident_wrong": 2, "macro_f1": 0.8388355871169197, "confusion_matrix": { "false": { "false": 185, "true": 15, "__failed__": 0 }, "true": { "false": 49, "true": 151, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 81, "accuracy": 0.81, "wilson95": [ 0.7222115462093562, 0.8748524849023126 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 81, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.81, "brier": 0.30286279776679875, "nll": 0.48490529698514756, "ece10": 0.12652606501322633, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.801234438748823, "confusion_matrix": { "yes": { "yes": 51, "no": 3, "__failed__": 0 }, "no": { "yes": 16, "no": 30, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 59, "accuracy": 0.59, "wilson95": [ 0.4920143005341078, 0.6813268697886348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 59, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.59, "brier": 0.579849569947826, "nll": 1.3682280486965794, "ece10": 0.07536757380431143, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.6008085664335665, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 4, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 6, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 5, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 5, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 6, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 4, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 7, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 4, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 4, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 143, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.715, "brier": 0.3728459715696838, "nll": 0.6070033683724015, "ece10": 0.12210132174466942, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 134, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.67, "brier": 0.4535490027596117, "nll": 0.8942068254221209, "ece10": 0.03914290542988092, "calibration_n": 200, "confident_wrong": 2 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.31350568596179684, "nll": 0.7806714613935186, "ece10": 0.05683705675187248, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.7439629270511623, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 13, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 12, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 1, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 7, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 6, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 6, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 1, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 220, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.6775720351881237, "nll": 1.3680162982208288, "ece10": 0.21416612045648495, "calibration_n": 400, "confident_wrong": 42 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 67, "accuracy": 0.67, "wilson95": [ 0.5730534379509052, 0.7543687726587197 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 67, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.67, "brier": 0.48893706894393324, "nll": 1.1219337290421008, "ece10": 0.09265499714688741, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6718214632916477, "confusion_matrix": { "politics": { "politics": 3, "business": 0, "technology": 2, "science": 2, "health": 3, "sports": 0, "entertainment": 1, "world": 2, "culture": 0, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 2, "technology": 7, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 2, "technology": 1, "science": 6, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 6, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 0, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 7, "world": 0, "culture": 1, "weather": 0, "__failed__": 0 }, "world": { "politics": 1, "business": 1, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 5, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 1, "world": 0, "culture": 8, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 6, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 43, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43, "brier": 0.7049597117538429, "nll": 1.6413874411192695, "ece10": 0.04794223136148084, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.40832268464210386, "confusion_matrix": { "biology": { "biology": 7, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 2, "medicine": 8, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 1, "medicine": 0, "chemistry": 4, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 4, "medicine": 2, "chemistry": 1, "physics": 6, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 2, "medicine": 2, "chemistry": 2, "physics": 0, "computer_science": 2, "mathematics": 0, "earth_science": 2, "engineering": 5, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 1, "physics": 1, "computer_science": 1, "mathematics": 1, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 1, "medicine": 4, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 6, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 2, "earth_science": 0, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 5, "other": 0, "__failed__": 0 }, "other": { "biology": 2, "medicine": 0, "chemistry": 0, "physics": 2, "computer_science": 0, "mathematics": 2, "earth_science": 2, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 13, "accuracy": 0.13, "wilson95": [ 0.07757167427240512, 0.20980351440076428 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.48516395871250806, "nll": 0.9464639085613774, "ece10": 0.16339317161061462, "calibration_n": 100, "confident_wrong": 7 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 42, "accuracy": 0.42, "wilson95": [ 0.3279838267435473, 0.5179351329695703 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 132, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.66, "brier": 0.31948512778067795, "nll": 0.5679497577263986, "ece10": 0.16755774786856045, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 87, "accuracy": 0.87, "wilson95": [ 0.7901964855992357, 0.9224283257275948 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 87, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.24327089555536877, "nll": 0.46512210787496777, "ece10": 0.16611165750960535, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8699055330634279, "confusion_matrix": { "negative": { "negative": 35, "neutral": 1, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 7, "neutral": 26, "positive": 0, "__failed__": 0 }, "positive": { "negative": 0, "neutral": 5, "positive": 26, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 19, "accuracy": 0.19, "wilson95": [ 0.12514751509768735, 0.27778845379064376 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 109, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.545, "brier": 0.2289316255306149, "nll": 0.35523479600883456, "ece10": 0.07977147658598066, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 234, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.78, "brier": 0.3036823527527572, "nll": 0.6084683967865521, "ece10": 0.018250654216557673, "calibration_n": 300, "confident_wrong": 6 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 69, "accuracy": 0.69, "wilson95": [ 0.5937394428051057, 0.7722030278762395 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.69, "brier": 0.435801259825688, "nll": 1.2227651817945973, "ece10": 0.12274612093309654, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6489191713981631, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 1, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 2, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 5, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 2, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 4, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 3, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 4, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 2, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 5, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 6, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 2, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 44, "accuracy": 0.44, "wilson95": [ 0.34672026630160796, 0.5377189534832302 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 44, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.44, "brier": 0.7179921059037482, "nll": 1.6189245771766358, "ece10": 0.1352312968661385, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.4404621958501269, "confusion_matrix": { "billing": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 1, "account_access": 6, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 4, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 3, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 2, "integrations": 4, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 6, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 3, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 3, "pricing": 0, "data_privacy": 1, "other": 1, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 5, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 2, "account_access": 0, "shipping": 3, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 1, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 12, "accuracy": 0.12, "wilson95": [ 0.06999406437019488, 0.19812099426711421 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 154, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5133333333333333, "brier": 0.6131497223728715, "nll": 1.3554730400288473, "ece10": 0.09977387414228447, "calibration_n": 300, "confident_wrong": 1 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 67, "accuracy": 0.67, "wilson95": [ 0.5730534379509052, 0.7543687726587197 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 67, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.67, "brier": 0.4943545115867846, "nll": 1.1570518580496258, "ece10": 0.07049391375632469, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.6418221756457051, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 1, "change_flight": 5, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 1, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 5, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 7, "disruption": 1, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 6, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 4, "car": 2, "visa_docs": 1, "loyalty": 1, "disruption": 1, "other": 1, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.1625102980919626, "nll": 0.29088829770074176, "ece10": 0.06748331016314216, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8683996929589101, "confusion_matrix": { "business": { "business": 25, "science and technology": 0, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 9, "science and technology": 13, "sports": 1, "world": 2, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 87, "accuracy": 0.87, "wilson95": [ 0.7901964855992357, 0.9224283257275948 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 87, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.21507851841288617, "nll": 0.5048261941475953, "ece10": 0.07383694983749373, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8537698412698411, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 2, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 2, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 2, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 2, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 1, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 53, "accuracy": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 53, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.53, "brier": 0.6163090069411071, "nll": 1.2860500001838722, "ece10": 0.09724809071617313, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.5368247358680628, "confusion_matrix": { "anger": { "anger": 11, "fear": 2, "joy": 0, "love": 0, "sadness": 3, "surprise": 1, "__failed__": 0 }, "fear": { "anger": 2, "fear": 11, "joy": 0, "love": 0, "sadness": 4, "surprise": 0, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 7, "love": 4, "sadness": 3, "surprise": 2, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 3, "love": 5, "sadness": 9, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 2, "joy": 2, "love": 1, "sadness": 11, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 0, "joy": 2, "love": 0, "sadness": 5, "surprise": 8, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2512, "accuracy": 0.8521031207598372, "wilson95": [ 0.838830303940018, 0.8644594997167794 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2512, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8521031207598372, "brier": 0.21606120948897617, "nll": 0.5138738204694544, "ece10": 0.02068491742367142, "calibration_n": 2948, "confident_wrong": 61, "macro_f1": 0.8476099325801791, "confusion_matrix": { "alarm": { "alarm": 92, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 0, "audio": 53, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 2, "audio": 0, "calendar": 349, "cooking": 0, "datetime": 7, "email": 1, "general": 1, "iot": 0, "lists": 6, "music": 0, "news": 3, "play": 1, "qa": 10, "recommendation": 17, "social": 1, "takeaway": 1, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 67, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 95, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 0, "email": 228, "general": 6, "iot": 1, "lists": 12, "music": 0, "news": 0, "play": 0, "qa": 15, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 11, "cooking": 6, "datetime": 1, "email": 1, "general": 66, "iot": 5, "lists": 0, "music": 2, "news": 5, "play": 5, "qa": 55, "recommendation": 16, "social": 7, "takeaway": 1, "transport": 3, "weather": 3, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 6, "datetime": 1, "email": 0, "general": 1, "iot": 196, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 7, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 1, "email": 0, "general": 3, "iot": 3, "lists": 125, "music": 2, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 71, "news": 0, "play": 3, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 107, "play": 1, "qa": 13, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 8, "iot": 8, "lists": 1, "music": 40, "news": 2, "play": 318, "qa": 4, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 1, "lists": 0, "music": 1, "news": 4, "play": 1, "qa": 268, "recommendation": 3, "social": 1, "takeaway": 0, "transport": 3, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 5, "play": 2, "qa": 7, "recommendation": 80, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 3, "general": 10, "iot": 1, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 2, "recommendation": 0, "social": 87, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 51, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 116, "weather": 3, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 4, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 143, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2367, "accuracy": 0.8029172320217096, "wilson95": [ 0.7881672986123502, 0.816878746344798 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2367, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8029172320217096, "brier": 0.28642991615606683, "nll": 0.6929392300784297, "ece10": 0.023461070687835643, "calibration_n": 2948, "confident_wrong": 69, "macro_f1": 0.7910189087387107, "confusion_matrix": { "alarm": { "alarm": 91, "audio": 2, "calendar": 2, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 3, "audio": 43, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 9, "iot": 1, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 7, "audio": 0, "calendar": 337, "cooking": 1, "datetime": 9, "email": 2, "general": 7, "iot": 3, "lists": 3, "music": 0, "news": 5, "play": 0, "qa": 7, "recommendation": 14, "social": 2, "takeaway": 0, "transport": 1, "weather": 2, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 65, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 98, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 1, "datetime": 0, "email": 232, "general": 6, "iot": 2, "lists": 10, "music": 0, "news": 0, "play": 0, "qa": 11, "recommendation": 2, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 12, "cooking": 8, "datetime": 2, "email": 1, "general": 65, "iot": 7, "lists": 0, "music": 2, "news": 7, "play": 3, "qa": 51, "recommendation": 13, "social": 5, "takeaway": 0, "transport": 7, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 2, "audio": 5, "calendar": 0, "cooking": 7, "datetime": 0, "email": 1, "general": 2, "iot": 182, "lists": 2, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 11, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 1, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 3, "iot": 5, "lists": 126, "music": 1, "news": 0, "play": 1, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 3, "iot": 0, "lists": 3, "music": 67, "news": 0, "play": 5, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 1, "iot": 1, "lists": 1, "music": 1, "news": 100, "play": 0, "qa": 16, "recommendation": 1, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "play": { "alarm": 1, "audio": 11, "calendar": 0, "cooking": 1, "datetime": 4, "email": 0, "general": 12, "iot": 5, "lists": 6, "music": 76, "news": 4, "play": 256, "qa": 8, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 6, "iot": 1, "lists": 1, "music": 3, "news": 7, "play": 1, "qa": 256, "recommendation": 4, "social": 1, "takeaway": 0, "transport": 3, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 4, "play": 2, "qa": 10, "recommendation": 76, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 3, "general": 12, "iot": 3, "lists": 0, "music": 0, "news": 2, "play": 0, "qa": 4, "recommendation": 1, "social": 77, "takeaway": 1, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 4, "datetime": 1, "email": 0, "general": 0, "iot": 1, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 48, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 1, "email": 0, "general": 2, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 112, "weather": 2, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 3, "iot": 3, "lists": 0, "music": 0, "news": 2, "play": 0, "qa": 4, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 136, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 844, "accuracy": 0.7195225916453538, "wilson95": [ 0.6931298996603696, 0.7444821475189323 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 844, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7195225916453538, "cluster_bootstrap95": [ 0.6865621655316797, 0.7552925553332505 ], "independent_source_groups": 47, "brier": 0.4160133779372271, "nll": 0.7179110318717667, "ece10": 0.0732718023912531, "calibration_n": 1173, "confident_wrong": 4, "macro_f1": 0.6880838971111024, "confusion_matrix": { "negative": { "negative": 118, "neutral": 54, "positive": 10, "__failed__": 0 }, "neutral": { "negative": 79, "neutral": 464, "positive": 47, "__failed__": 0 }, "positive": { "negative": 34, "neutral": 105, "positive": 262, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "decision2-sol-2b": { "label": "Decision 2.0 Sol 2B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 167, "accuracy": 0.835, "wilson95": [ 0.7773421846094045, 0.8800314467178268 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 167, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.835, "brier": 0.23195408765393105, "nll": 0.4591380060613916, "ece10": 0.039382858870365425, "calibration_n": 200, "confident_wrong": 5, "macro_f1": 0.8358803069477517, "confusion_matrix": { "world": { "world": 46, "sports": 0, "business": 0, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 9, "sports": 0, "business": 41, "sci_tech": 1, "__failed__": 0 }, "sci_tech": { "world": 7, "sports": 1, "business": 10, "sci_tech": 34, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 58, "accuracy": 0.29, "wilson95": [ 0.23154025607067882, 0.3563747810077731 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.29, "brier": 0.8155000339764633, "nll": 1.739463382274664, "ece10": 0.07738324240289889, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.09528666751668154, "confusion_matrix": { "sadness": { "sadness": 56, "joy": 0, "love": 0, "anger": 1, "fear": 0, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 79, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 12, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 25, "joy": 0, "love": 0, "anger": 2, "fear": 0, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 20, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "surprise": { "sadness": 4, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 149, "accuracy": 0.745, "wilson95": [ 0.6803707417954529, 0.8003950482796865 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 149, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.745, "brier": 0.3403342681089129, "nll": 0.5098864754974484, "ece10": 0.06877994921058281, "calibration_n": 200, "confident_wrong": 4, "macro_f1": 0.7439180537772088, "confusion_matrix": { "false": { "false": 81, "true": 28, "__failed__": 0 }, "true": { "false": 23, "true": 68, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 172, "accuracy": 0.86, "wilson95": [ 0.8051011913429243, 0.9013301736654437 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 172, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.23538100457286404, "nll": 0.39377373803301446, "ece10": 0.05861484046555323, "calibration_n": 200, "confident_wrong": 5, "macro_f1": 0.8514431239388794, "confusion_matrix": { "true": { "true": 62, "false": 15, "__failed__": 0 }, "false": { "true": 13, "false": 110, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 7, "accuracy": 0.5833333333333334, "wilson95": [ 0.31951131254954973, 0.8067396863412435 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5833333333333334, "brier": 0.42842671963849815, "nll": 0.7095321209502953, "ece10": 0.2311789664267936, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.5555555555555556, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 7, "accuracy": 0.5833333333333334, "wilson95": [ 0.31951131254954973, 0.8067396863412435 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5833333333333334, "brier": 0.5830241232759327, "nll": 1.1851247227428805, "ece10": 0.12113809839010514, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.5504761904761905, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 2, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.022178503123876214, "nll": 0.0891425708769062, "ece10": 0.0824487973525268, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.05248382037510426, "nll": 0.15518418716809673, "ece10": 0.13675118534536743, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.3094013777239084, "nll": 0.5573990066870256, "ece10": 0.07808215492913945, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.7683881335553089, "confusion_matrix": { "label_000": { "label_000": 24, "label_001": 0, "label_002": 0, "label_003": 1, "__failed__": 0 }, "label_001": { "label_000": 14, "label_001": 7, "label_002": 1, "label_003": 3, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 24, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 52, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.62230564883843, "nll": 1.3518112630730417, "ece10": 0.15807219682167148, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.4995189534035706, "confusion_matrix": { "label_000": { "label_000": 14, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 1, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 14, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 7, "label_003": 5, "label_004": 2, "label_005": 2, "__failed__": 0 }, "label_003": { "label_000": 4, "label_001": 0, "label_002": 3, "label_003": 6, "label_004": 4, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 2, "label_001": 3, "label_002": 3, "label_003": 1, "label_004": 7, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 4, "label_001": 1, "label_002": 4, "label_003": 0, "label_004": 3, "label_005": 4, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 89, "accuracy": 0.89, "wilson95": [ 0.8136870349691969, 0.9374580364293543 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 89, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.89, "brier": 0.19393854356598175, "nll": 0.4859641467382915, "ece10": 0.12197154027504237, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8837962962962963, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 1, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 1, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 2, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 2, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 2, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 2, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 1, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.002178764715418645, "nll": 0.018622764241604303, "ece10": 0.01791468202690033, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 51, "accuracy": 0.4594594594594595, "wilson95": [ 0.36965969632385665, 0.5519713932458781 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 51, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4594594594594595, "brier": 0.6233920897918158, "nll": 1.072830678037764, "ece10": 0.15873165441089904, "calibration_n": 111, "confident_wrong": 4 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 67, "accuracy": 0.9305555555555556, "wilson95": [ 0.8475202051632649, 0.9699746184342299 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 67, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9305555555555556, "cluster_bootstrap95": [ 0.8472222222222222, 0.9861111111111112 ], "independent_source_groups": 36, "brier": 0.15628607397146663, "nll": 0.31196000404227486, "ece10": 0.11232565664760107, "calibration_n": 72, "confident_wrong": 1 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 17, "accuracy": 0.8947368421052632, "wilson95": [ 0.6860591728889099, 0.9706414397011299 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8947368421052632, "brier": 0.1570492986923183, "nll": 0.26091003320247885, "ece10": 0.06563985990132665, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8944444444444444, "confusion_matrix": { "false": { "false": 9, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.11021696629620746, "nll": 0.19371927979717507, "ece10": 0.1506603818381906, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9372549019607843, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 15, "accuracy": 0.6818181818181818, "wilson95": [ 0.47318597741042745, 0.8363940681864375 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6818181818181818, "brier": 0.4367559236500398, "nll": 0.6524815201141861, "ece10": 0.2035040538646518, "calibration_n": 22, "confident_wrong": 3, "macro_f1": 0.6644880174291938, "confusion_matrix": { "false": { "false": 10, "true": 0, "__failed__": 0 }, "true": { "false": 7, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.08285766616105902, "nll": 0.1689000657565872, "ece10": 0.13044388852010144, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9415584415584415, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 3, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.39220824345426625, "nll": 0.7280873294935094, "ece10": 0.20602959754045272, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6652813852813853, "confusion_matrix": { "0": { "0": 1, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 3, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 2, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.01568451189446544, "nll": 0.07134880465236948, "ece10": 0.06603468985503552, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.24061777841549883, "nll": 0.5090601145029967, "ece10": 0.3277148692381538, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9428571428571427, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 1, "security": 0, "style": 0, "refactor": 0, "nit": 3, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 11, "accuracy": 0.7333333333333333, "wilson95": [ 0.4804956594401943, 0.8910254667430761 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7333333333333333, "brier": 0.41372070503579367, "nll": 0.7396765480554705, "ece10": 0.2838062526007478, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7166666666666667, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 1, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 1, "refactor": 1, "docs": 0, "test": 0, "chore": 2, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.16096263073412556, "nll": 0.24581172597571394, "ece10": 0.17477436430606016, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8235294117647058, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.38628076286529267, "nll": 0.5305505086228997, "ece10": 0.2388289723632821, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5465587044534412, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 5, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.39608093874289074, "nll": 0.8530602536382982, "ece10": 0.19125649653955706, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5494949494949495, "confusion_matrix": { "harassment": { "harassment": 5, "threat": 0, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 1, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 1, "threat": 0, "self_harm": 0, "spam": 3, "none": 2, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.19928362753794196, "nll": 0.4119709615218905, "ece10": 0.1628845855151937, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9092063492063491, "confusion_matrix": { "news_report": { "news_report": 4, "opinion": 0, "press_release": 1, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 1, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 3, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03619103942094065, "nll": 0.09293652938435594, "ece10": 0.07935802582385537, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.13173809162533012, "nll": 0.30014215642019226, "ece10": 0.1984888247290693, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "address_issue": { "address_issue": 3, "customs_hold": 1, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.31448186343440354, "nll": 0.45744621187718926, "ece10": 0.21478956094588997, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6357142857142857, "confusion_matrix": { "false": { "false": 7, "true": 2, "__failed__": 0 }, "true": { "false": 4, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.11540322791125522, "nll": 0.23860178404720042, "ece10": 0.16253156425844623, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 1, "receipt": 2, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.1442416146626536, "nll": 0.35134616117829537, "ece10": 0.10155326177556626, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.15151797175895074, "nll": 0.28299079703640073, "ece10": 0.1724544256750453, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.15620553536969312, "nll": 0.2763536606321083, "ece10": 0.1213632781779712, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9442724458204335, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.5785292348991593, "nll": 0.8654955283605048, "ece10": 0.2102188651926319, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.4797979797979798, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.43049679932269486, "nll": 0.7231310546498176, "ece10": 0.13449622741682532, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7037037037037037, "confusion_matrix": { "0": { "0": 1, "1": 4, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 18, "accuracy": 0.9, "wilson95": [ 0.6989663547715128, 0.9721335187862319 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.25124152104227676, "nll": 0.5693039120184086, "ece10": 0.2625425929206903, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9095238095238095, "confusion_matrix": { "cheating": { "cheating": 3, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 4, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 3, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 7, "accuracy": 0.3333333333333333, "wilson95": [ 0.17194752605083477, 0.5462654802574476 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3333333333333333, "brier": 0.8926844960043014, "nll": 1.88917450915684, "ece10": 0.28487473555424964, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.2051948051948052, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 0, "punctuation": 0, "none": 1, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 0, "punctuation": 0, "none": 2, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.17488244210572296, "nll": 0.26743168720729393, "ece10": 0.08794624990371827, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.875, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.027989618026008304, "nll": 0.11175320909975052, "ece10": 0.09918918540008549, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 13, "accuracy": 0.9285714285714286, "wilson95": [ 0.6853129557584889, 0.9872777847521093 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.3877385101878425, "nll": 0.7547719718944439, "ece10": 0.43380332161757623, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.9047619047619048, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 1, "4": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.43775690791835054, "nll": 0.8181360110444933, "ece10": 0.33732233395948874, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6031746031746031, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 1, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 1, "2": 1, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.46615731579280495, "nll": 0.824038787507804, "ece10": 0.19676181417513888, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6597222222222222, "confusion_matrix": { "0": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.41420406131819365, "nll": 0.5922078930387829, "ece10": 0.11729038538124288, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6136363636363636, "confusion_matrix": { "false": { "false": 3, "true": 5, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.11077337828157838, "nll": 0.23289796777257105, "ece10": 0.10716554914163132, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9377777777777778, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "sports": 2, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.12100744339357188, "nll": 0.24257625461223925, "ece10": 0.07839248311684321, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "payments": { "payments": 3, "web": 0, "infra": 1, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.2838543372590438, "nll": 0.4199587091426284, "ece10": 0.13466970765336722, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7638888888888888, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.1327953273531051, "nll": 0.26263661517668735, "ece10": 0.21850877362755844, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.2858386325436252, "nll": 0.5405948057727731, "ece10": 0.23094745713163556, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8055555555555557, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 1, "__failed__": 0 }, "2": { "0": 1, "1": 0, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.3498751121626408, "nll": 0.6791776402467963, "ece10": 0.1871289101483347, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7502645502645503, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 3, "french": 1, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 1, "french": 0, "mexican": 2, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.4405583296022335, "nll": 0.6505964450168081, "ece10": 0.1712194337671232, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6640316205533596, "confusion_matrix": { "false": { "false": 3, "true": 5, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.23822482117892832, "nll": 0.4700868284090862, "ece10": 0.2152285607527212, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8133333333333332, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 2, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.5742386417998634, "nll": 1.1923791814092577, "ece10": 0.17474285146905266, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6666666666666666, "confusion_matrix": { "0": { "0": 1, "1": 2, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 3, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.516946802780204, "nll": 0.7026428387659307, "ece10": 0.2736881467176969, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6190476190476191, "confusion_matrix": { "false": { "false": 4, "true": 6, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 8, "accuracy": 0.5333333333333333, "wilson95": [ 0.30116980025498397, 0.7519046463426261 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5333333333333333, "brier": 0.785581081409563, "nll": 1.28704756415351, "ece10": 0.4408154154532222, "calibration_n": 15, "confident_wrong": 4, "macro_f1": 0.4444444444444444, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.3950545563024561, "nll": 0.6721542465111489, "ece10": 0.16406165310137266, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.7692063492063491, "confusion_matrix": { "billing": { "billing": 4, "tech": 0, "sales": 0, "account": 0, "other": 1, "__failed__": 0 }, "tech": { "billing": 0, "tech": 2, "sales": 0, "account": 0, "other": 1, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 3, "account": 0, "other": 1, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.324682295996559, "nll": 0.49414510789345784, "ece10": 0.1253003684572357, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7571428571428571, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 11, "accuracy": 0.5789473684210527, "wilson95": [ 0.3627590353425504, 0.7685810871754576 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5789473684210527, "brier": 0.43599783779250506, "nll": 0.7614890100670606, "ece10": 0.20394558880022526, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.506827731092437, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 4, "1": 0, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 2, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.46839354752049056, "nll": 0.6516872085126998, "ece10": 0.2557089470645456, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6136363636363636, "confusion_matrix": { "false": { "false": 8, "true": 3, "__failed__": 0 }, "true": { "false": 3, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.059550820922907505, "nll": 0.14180082216070686, "ece10": 0.12249678896927066, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 12, "accuracy": 0.6, "wilson95": [ 0.3865815007622531, 0.781193467627183 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5159974656144962, "nll": 0.9315005605230657, "ece10": 0.24645428090414012, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.4984848484848485, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 2, "3": 0, "__failed__": 0 }, "1": { "0": 3, "1": 0, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.05113515362258122, "nll": 0.12105040371862974, "ece10": 0.09547123879480007, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 12, "accuracy": 0.6, "wilson95": [ 0.3865815007622531, 0.781193467627183 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5624700705479116, "nll": 0.8040298427105922, "ece10": 0.2721247521168513, "calibration_n": 20, "confident_wrong": 2, "macro_f1": 0.5604395604395604, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 8, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 16, "accuracy": 0.7272727272727273, "wilson95": [ 0.5184826598759762, 0.8684923971201053 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7272727272727273, "brier": 0.43599189236146557, "nll": 0.8278405087568608, "ece10": 0.24920746402453142, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.7236928104575164, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 1, "__failed__": 0 }, "1": { "0": 1, "1": 2, "2": 0, "3": 3, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 512, "accuracy": 0.5688888888888889, "wilson95": [ 0.5363090668387911, 0.6008831351672903 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 656, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6113699906803355, "cluster_bootstrap95": [ 0.5335806257585113, 0.6041116282034935 ], "independent_source_groups": 596, "brier": 0.49018929752741897, "nll": 0.8459950999250038, "ece10": 0.11658865245955177, "calibration_n": 1073, "confident_wrong": 35 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 300, "accuracy": 0.5226480836236934, "wilson95": [ 0.48177161704415983, 0.5632234237288974 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 622, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6645299145299145, "brier": 0.4495987464764802, "nll": 0.8766754831510221, "ece10": 0.04886389905633808, "calibration_n": 936, "confident_wrong": 2 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 161, "accuracy": 0.23, "wilson95": [ 0.20034876983953173, 0.26259846740712117 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 396, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3639705882352941, "cluster_bootstrap95": [ 0.19913307267367447, 0.26149698093990975 ], "independent_source_groups": 650, "brier": 0.721959478803904, "nll": 1.3996544530480595, "ece10": 0.14805894497470545, "calibration_n": 1088, "confident_wrong": 13 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 25, "accuracy": 0.0625, "wilson95": [ 0.04268914141220244, 0.09063411613030699 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1102, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.551, "brier": 0.555920117850975, "nll": 0.9721889346854986, "ece10": 0.05980603739468886, "calibration_n": 2000, "confident_wrong": 28 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.021222758396229637, "nll": 0.08286603935732444, "ece10": 0.07568117735070634, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.05283006751609493, "nll": 0.14136337136571012, "ece10": 0.11943105964889364, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "work": { "work": 6, "purchase": 0, "inspection": 0, "design": 1, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.023868708079755625, "nll": 0.09772102117939074, "ece10": 0.08926798044883771, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.08984177103004642, "nll": 0.19665336989623516, "ece10": 0.121581326945683, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 1, "purchase": 0, "inspection": 0, "design": 6, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 31, "accuracy": 0.775, "wilson95": [ 0.6249690325768998, 0.8768390867645927 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 31, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.775, "brier": 0.2856541036449992, "nll": 0.5211861259419688, "ece10": 0.16349289143399845, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7725763125763125, "confusion_matrix": { "review": { "review": 4, "it": 0, "documents": 0, "procurement": 1, "maintenance": 3, "__failed__": 0 }, "it": { "review": 1, "it": 5, "documents": 0, "procurement": 0, "maintenance": 2, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 7, "procurement": 1, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 1, "it": 0, "documents": 0, "procurement": 0, "maintenance": 7, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.24497288669321415, "nll": 0.48035796507998596, "ece10": 0.19371464405577096, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8804636591478696, "confusion_matrix": { "maintenance": { "maintenance": 7, "procurement": 0, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 7, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 6, "it": 0, "review": 2, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 28, "accuracy": 0.7, "wilson95": [ 0.5456998118185507, 0.8192515477025347 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.40460690387556814, "nll": 0.7756178804563524, "ece10": 0.1413429391457714, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7047619047619047, "confusion_matrix": { "review": { "review": 5, "it": 0, "documents": 1, "procurement": 0, "maintenance": 2, "__failed__": 0 }, "it": { "review": 1, "it": 4, "documents": 1, "procurement": 0, "maintenance": 2, "__failed__": 0 }, "documents": { "review": 1, "it": 0, "documents": 7, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 1, "it": 0, "documents": 1, "procurement": 6, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 2, "it": 0, "documents": 0, "procurement": 0, "maintenance": 6, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 30, "accuracy": 0.75, "wilson95": [ 0.5980603857923197, 0.858128813609037 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 30, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.403810422323773, "nll": 0.772379775546643, "ece10": 0.13051176375969464, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7588461538461538, "confusion_matrix": { "maintenance": { "maintenance": 6, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 5, "documents": 1, "it": 0, "review": 2, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 7, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 6, "review": 1, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 1, "review": 6, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 126, "accuracy": 0.315, "wilson95": [ 0.2714209156648816, 0.3620986332388081 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 126, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.315, "brier": 0.8115939551891662, "nll": 1.7287751113526508, "ece10": 0.10085037181619051, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.11412583916058205, "confusion_matrix": { "sadness": { "sadness": 119, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 118, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 37, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 58, "joy": 0, "love": 0, "anger": 5, "fear": 0, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 50, "joy": 0, "love": 0, "anger": 0, "fear": 2, "surprise": 0, "__failed__": 0 }, "surprise": { "sadness": 10, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 254, "accuracy": 0.8466666666666667, "wilson95": [ 0.8015339429418983, 0.8830335970090831 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 254, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8466666666666667, "brier": 0.194681366829805, "nll": 0.46944689958220676, "ece10": 0.05422101177321817, "calibration_n": 300, "confident_wrong": 1 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 367, "accuracy": 0.9175, "wilson95": [ 0.8864031104605777, 0.9406541237702847 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 367, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9175, "brier": 0.1332102475244129, "nll": 0.23798443071459585, "ece10": 0.06758592205181284, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.8856935027147793, "confusion_matrix": { "false": { "false": 289, "true": 20, "__failed__": 0 }, "true": { "false": 13, "true": 78, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 328, "accuracy": 0.82, "wilson95": [ 0.7793624469518832, 0.8545496846741669 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 328, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.82, "brier": 0.2684407259700471, "nll": 0.5047132564092944, "ece10": 0.07587187458735661, "calibration_n": 400, "confident_wrong": 18, "macro_f1": 0.8029231453826189, "confusion_matrix": { "world": { "world": 88, "sports": 6, "business": 8, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 3, "sports": 120, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 7, "sports": 1, "business": 63, "sci_tech": 1, "__failed__": 0 }, "sci_tech": { "world": 16, "sports": 0, "business": 29, "sci_tech": 57, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 341, "accuracy": 0.8525, "wilson95": [ 0.8143999058527144, 0.8838939266416067 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 341, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8525, "brier": 0.22233104026828943, "nll": 0.3642882590594764, "ece10": 0.057849737551275746, "calibration_n": 400, "confident_wrong": 4, "macro_f1": 0.8471255060728745, "confusion_matrix": { "true": { "true": 133, "false": 20, "__failed__": 0 }, "false": { "true": 39, "false": 208, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 217, "accuracy": 0.7233333333333334, "wilson95": [ 0.6701300560341223, 0.7708894168189138 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 217, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7233333333333334, "brier": 0.3922997052768055, "nll": 0.9729522203355204, "ece10": 0.06595443036733628, "calibration_n": 300, "confident_wrong": 6 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 320, "accuracy": 0.8, "wilson95": [ 0.758029683133299, 0.8362629402661231 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 320, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.27387719503544183, "nll": 0.4385458428199341, "ece10": 0.046561777291190605, "calibration_n": 400, "confident_wrong": 16, "macro_f1": 0.7993931643220743, "confusion_matrix": { "false": { "false": 149, "true": 39, "__failed__": 0 }, "true": { "false": 41, "true": 171, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 316, "accuracy": 0.79, "wilson95": [ 0.7474205944599498, 0.8270622748261584 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 316, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.3258298909494575, "nll": 0.5015724739985109, "ece10": 0.05585638233640426, "calibration_n": 400, "confident_wrong": 4, "macro_f1": 0.7892412685668406, "confusion_matrix": { "false": { "false": 170, "true": 30, "__failed__": 0 }, "true": { "false": 54, "true": 146, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 75, "accuracy": 0.75, "wilson95": [ 0.656955364519384, 0.8245478863771232 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 75, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3660810709197846, "nll": 0.5523485389139613, "ece10": 0.059437624941391695, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.7333333333333334, "confusion_matrix": { "yes": { "yes": 50, "no": 4, "__failed__": 0 }, "no": { "yes": 21, "no": 25, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 55, "accuracy": 0.55, "wilson95": [ 0.45244602997442135, 0.6438546202048803 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 55, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.6015201792387319, "nll": 1.4143254092869557, "ece10": 0.09224661824844109, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5357259215635175, "confusion_matrix": { "balance": { "balance": 3, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 3, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 4, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 3, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 1, "card_lost": 1, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 1, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 2, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 4, "branch_appointment": 2, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 6, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 1, "fee_refund": 9, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 4, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 5, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 2, "branch_appointment": 2, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 50, "accuracy": 0.5, "wilson95": [ 0.4038315303659956, 0.5961684696340044 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 144, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.72, "brier": 0.384265182111765, "nll": 0.653403457044351, "ece10": 0.0815326922671254, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 38, "accuracy": 0.38, "wilson95": [ 0.29097599252478734, 0.47790244704488916 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 125, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.47832212599657997, "nll": 0.9468571624053372, "ece10": 0.07340707112581585, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.34033843666696106, "nll": 0.8963735907080845, "ece10": 0.08729988193788625, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.7491205500133118, "confusion_matrix": { "invoice": { "invoice": 6, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 14, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 13, "clinical_note": 0, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 7, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 1, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 6, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 5, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 1, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 7, "accuracy": 0.07, "wilson95": [ 0.03431926106727268, 0.137495147390735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 206, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.515, "brier": 0.7049394635209564, "nll": 1.3970541963541085, "ece10": 0.22052586147484465, "calibration_n": 400, "confident_wrong": 51 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.5509757939815427, "nll": 1.2506581508189618, "ece10": 0.13386536972081378, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5622239619311904, "confusion_matrix": { "politics": { "politics": 3, "business": 0, "technology": 1, "science": 3, "health": 5, "sports": 0, "entertainment": 1, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 2, "technology": 7, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 1, "business": 2, "technology": 0, "science": 3, "health": 4, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 6, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 7, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "world": { "politics": 2, "business": 2, "technology": 1, "science": 0, "health": 2, "sports": 0, "entertainment": 1, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "culture": { "politics": 2, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 1, "world": 0, "culture": 6, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 7, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 41, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.41, "brier": 0.7357031221127025, "nll": 1.7279286423839284, "ece10": 0.08078339929395337, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.38605548025661884, "confusion_matrix": { "biology": { "biology": 5, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 0, "engineering": 1, "economics": 0, "other": 1, "__failed__": 0 }, "medicine": { "biology": 0, "medicine": 5, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 2, "earth_science": 3, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 1, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 2, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 2, "medicine": 1, "chemistry": 0, "physics": 6, "computer_science": 1, "mathematics": 0, "earth_science": 3, "engineering": 3, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 6, "mathematics": 0, "earth_science": 1, "engineering": 7, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 1, "physics": 1, "computer_science": 2, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 8, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 2, "earth_science": 0, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 0, "engineering": 1, "economics": 6, "other": 0, "__failed__": 0 }, "other": { "biology": 1, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 2, "earth_science": 2, "engineering": 5, "economics": 1, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 3, "accuracy": 0.03, "wilson95": [ 0.010254524024038925, 0.0845193642905276 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 68, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.34, "brier": 0.5069165762092338, "nll": 1.04659071831906, "ece10": 0.08963781350552531, "calibration_n": 100, "confident_wrong": 4 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 19, "accuracy": 0.19, "wilson95": [ 0.12514751509768735, 0.27778845379064376 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 102, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.51, "brier": 0.3631882349547358, "nll": 0.6251928320453594, "ece10": 0.09250068984968213, "calibration_n": 100, "confident_wrong": 2 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 76, "accuracy": 0.76, "wilson95": [ 0.6676766365018553, 0.8330867444305122 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 76, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.35173023530399233, "nll": 0.588524273025591, "ece10": 0.10542693973034116, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.7498889066638226, "confusion_matrix": { "negative": { "negative": 36, "neutral": 0, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 19, "neutral": 14, "positive": 0, "__failed__": 0 }, "positive": { "negative": 3, "neutral": 2, "positive": 26, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 18, "accuracy": 0.18, "wilson95": [ 0.1170022549894635, 0.2666735838630073 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 100, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.27632859578900426, "nll": 0.41820469926237286, "ece10": 0.09269865931462723, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 44, "accuracy": 0.44, "wilson95": [ 0.34672026630160796, 0.5377189534832302 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 229, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7633333333333333, "brier": 0.3509157403784563, "nll": 0.6908462114000777, "ece10": 0.025166218706536143, "calibration_n": 300, "confident_wrong": 4 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 55, "accuracy": 0.55, "wilson95": [ 0.45244602997442135, 0.6438546202048803 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 55, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.5918594055699193, "nll": 1.5428990434081387, "ece10": 0.1501719737746777, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.54027305560919, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 1, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 1, "password_reset": 3, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 1, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 4, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 3, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 2, "complaint": 3, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 3, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 4, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 5, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 1, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 3, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 49, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.7128276173154156, "nll": 1.7116323224233083, "ece10": 0.07835742492090617, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.491749222999223, "confusion_matrix": { "billing": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 2, "__failed__": 0 }, "account_access": { "billing": 0, "account_access": 6, "shipping": 1, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 5, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 5, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 2, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 4, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 0, "integrations": 4, "security": 1, "compliance": 2, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 5, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 1, "pricing": 1, "data_privacy": 1, "other": 1, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 4, "data_privacy": 1, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 4, "other": 0, "__failed__": 0 }, "other": { "billing": 2, "account_access": 0, "shipping": 2, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 2, "pricing": 0, "data_privacy": 1, "other": 1, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 11, "accuracy": 0.11, "wilson95": [ 0.06254196357064568, 0.18631296503080313 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 147, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.6450734140324357, "nll": 1.4148486444110904, "ece10": 0.05860776969173617, "calibration_n": 300, "confident_wrong": 3 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 66, "accuracy": 0.66, "wilson95": [ 0.5627772885472462, 0.7453847920265184 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 66, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.66, "brier": 0.5321099923317381, "nll": 1.1624570284859057, "ece10": 0.12051706228728476, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6390741913458633, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 3, "change_flight": 6, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 11, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 7, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 7, "car": 0, "visa_docs": 1, "loyalty": 0, "disruption": 2, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 6, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 1, "loyalty": 6, "disruption": 1, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 7, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 1, "hotel": 0, "car": 1, "visa_docs": 1, "loyalty": 3, "disruption": 5, "other": 1, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.24878691541438855, "nll": 0.4346898162552887, "ece10": 0.06880886073700875, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.7854016550825061, "confusion_matrix": { "business": { "business": 25, "science and technology": 0, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 12, "science and technology": 10, "sports": 1, "world": 2, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 4, "science and technology": 0, "sports": 1, "world": 20, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 90, "accuracy": 0.9, "wilson95": [ 0.8256343384950865, 0.9447708629393249 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 90, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.19889295141911462, "nll": 0.48215294500488, "ece10": 0.1282195452476215, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8912037037037036, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 2, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 2, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 2, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 2, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 2, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 1, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 1, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 49, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.6431851926066946, "nll": 1.3408790449148171, "ece10": 0.207736097870836, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.4889382783404523, "confusion_matrix": { "anger": { "anger": 12, "fear": 2, "joy": 0, "love": 0, "sadness": 3, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 1, "fear": 11, "joy": 0, "love": 0, "sadness": 5, "surprise": 0, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 7, "love": 4, "sadness": 4, "surprise": 2, "__failed__": 0 }, "love": { "anger": 2, "fear": 0, "joy": 3, "love": 6, "sadness": 6, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 2, "fear": 4, "joy": 1, "love": 1, "sadness": 8, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 3, "fear": 1, "joy": 4, "love": 0, "sadness": 3, "surprise": 5, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2165, "accuracy": 0.7343962008141113, "wilson95": [ 0.7181557430936916, 0.7500265828473942 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2165, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7343962008141113, "brier": 0.38142632404435634, "nll": 0.8983310605498289, "ece10": 0.04883348534706849, "calibration_n": 2948, "confident_wrong": 86, "macro_f1": 0.7246696078366311, "confusion_matrix": { "alarm": { "alarm": 64, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 8, "email": 0, "general": 0, "iot": 21, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 40, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 8, "iot": 10, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 37, "audio": 0, "calendar": 196, "cooking": 0, "datetime": 79, "email": 5, "general": 6, "iot": 1, "lists": 37, "music": 0, "news": 9, "play": 0, "qa": 7, "recommendation": 18, "social": 3, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 66, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 81, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 19, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 3, "email": 221, "general": 8, "iot": 0, "lists": 20, "music": 0, "news": 1, "play": 0, "qa": 9, "recommendation": 1, "social": 3, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 7, "datetime": 6, "email": 6, "general": 84, "iot": 7, "lists": 0, "music": 1, "news": 7, "play": 5, "qa": 27, "recommendation": 14, "social": 5, "takeaway": 4, "transport": 5, "weather": 3, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 16, "datetime": 0, "email": 0, "general": 5, "iot": 187, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 5, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 1, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 2, "email": 0, "general": 1, "iot": 5, "lists": 128, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 2, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 17, "iot": 0, "lists": 6, "music": 45, "news": 0, "play": 3, "qa": 2, "recommendation": 1, "social": 2, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 5, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 1, "lists": 0, "music": 0, "news": 109, "play": 0, "qa": 5, "recommendation": 1, "social": 1, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 2, "calendar": 0, "cooking": 0, "datetime": 2, "email": 4, "general": 23, "iot": 12, "lists": 6, "music": 38, "news": 1, "play": 293, "qa": 0, "recommendation": 2, "social": 2, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 6, "email": 2, "general": 13, "iot": 7, "lists": 0, "music": 1, "news": 10, "play": 1, "qa": 231, "recommendation": 5, "social": 3, "takeaway": 0, "transport": 2, "weather": 6, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 3, "email": 0, "general": 6, "iot": 2, "lists": 0, "music": 0, "news": 13, "play": 2, "qa": 3, "recommendation": 65, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 6, "general": 6, "iot": 1, "lists": 0, "music": 0, "news": 5, "play": 0, "qa": 0, "recommendation": 0, "social": 87, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 2, "datetime": 1, "email": 0, "general": 1, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 3, "social": 0, "takeaway": 48, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 19, "email": 0, "general": 2, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 7, "recommendation": 10, "social": 0, "takeaway": 1, "transport": 80, "weather": 2, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 4, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 2, "weather": 140, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1898, "accuracy": 0.6438263229308006, "wilson95": [ 0.6263631846040778, 0.6609151166969647 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1898, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6438263229308006, "brier": 0.4862412561185914, "nll": 1.1862859734825655, "ece10": 0.06535133300598191, "calibration_n": 2948, "confident_wrong": 63, "macro_f1": 0.6461877179332203, "confusion_matrix": { "alarm": { "alarm": 54, "audio": 1, "calendar": 1, "cooking": 0, "datetime": 5, "email": 0, "general": 2, "iot": 33, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 2, "audio": 30, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 12, "iot": 13, "lists": 1, "music": 0, "news": 0, "play": 1, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 34, "audio": 0, "calendar": 141, "cooking": 1, "datetime": 84, "email": 15, "general": 20, "iot": 3, "lists": 51, "music": 0, "news": 8, "play": 0, "qa": 5, "recommendation": 28, "social": 5, "takeaway": 2, "transport": 2, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 65, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 83, "email": 0, "general": 1, "iot": 6, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 8, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 4, "email": 225, "general": 13, "iot": 1, "lists": 12, "music": 0, "news": 0, "play": 0, "qa": 8, "recommendation": 2, "social": 3, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 1, "audio": 0, "calendar": 8, "cooking": 8, "datetime": 9, "email": 6, "general": 82, "iot": 8, "lists": 2, "music": 3, "news": 8, "play": 2, "qa": 17, "recommendation": 12, "social": 9, "takeaway": 1, "transport": 4, "weather": 7, "__failed__": 0 }, "iot": { "alarm": 3, "audio": 1, "calendar": 0, "cooking": 10, "datetime": 0, "email": 2, "general": 7, "iot": 177, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 1, "takeaway": 10, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 1, "general": 7, "iot": 4, "lists": 123, "music": 1, "news": 0, "play": 2, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 1, "email": 1, "general": 13, "iot": 0, "lists": 5, "music": 49, "news": 2, "play": 6, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 2, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 3, "general": 4, "iot": 0, "lists": 1, "music": 1, "news": 100, "play": 0, "qa": 7, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "play": { "alarm": 7, "audio": 3, "calendar": 0, "cooking": 3, "datetime": 3, "email": 4, "general": 51, "iot": 21, "lists": 26, "music": 50, "news": 8, "play": 194, "qa": 3, "recommendation": 2, "social": 8, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "qa": { "alarm": 2, "audio": 1, "calendar": 0, "cooking": 2, "datetime": 8, "email": 6, "general": 26, "iot": 10, "lists": 2, "music": 2, "news": 11, "play": 1, "qa": 200, "recommendation": 7, "social": 2, "takeaway": 1, "transport": 2, "weather": 4, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 4, "email": 0, "general": 9, "iot": 3, "lists": 0, "music": 0, "news": 7, "play": 3, "qa": 1, "recommendation": 61, "social": 1, "takeaway": 1, "transport": 2, "weather": 0, "__failed__": 0 }, "social": { "alarm": 1, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 10, "general": 7, "iot": 2, "lists": 0, "music": 0, "news": 6, "play": 0, "qa": 0, "recommendation": 0, "social": 77, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 2, "email": 1, "general": 1, "iot": 3, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 47, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 18, "email": 4, "general": 3, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 11, "social": 0, "takeaway": 6, "transport": 73, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 10, "email": 1, "general": 9, "iot": 5, "lists": 0, "music": 2, "news": 2, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 2, "weather": 117, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 671, "accuracy": 0.5720375106564365, "wilson95": [ 0.5435327924347861, 0.6000719376104604 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 671, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5720375106564365, "cluster_bootstrap95": [ 0.5466143269816716, 0.5982461210830544 ], "independent_source_groups": 47, "brier": 0.5329661644692806, "nll": 0.8993872379238508, "ece10": 0.054393353894771776, "calibration_n": 1173, "confident_wrong": 4, "macro_f1": 0.5618515858531928, "confusion_matrix": { "negative": { "negative": 132, "neutral": 31, "positive": 19, "__failed__": 0 }, "neutral": { "negative": 224, "neutral": 265, "positive": 101, "__failed__": 0 }, "positive": { "negative": 70, "neutral": 57, "positive": 274, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "gliner-decide-1b": { "label": "GLiNER Decide 1B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 142, "accuracy": 0.71, "wilson95": [ 0.6436252189922268, 0.7684597439293211 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 142, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.71, "brier": 0.4192196058853353, "nll": 0.8387304984113478, "ece10": 0.10435980685289464, "calibration_n": 200, "confident_wrong": 14, "macro_f1": 0.7073736662151296, "confusion_matrix": { "world": { "world": 45, "sports": 1, "business": 3, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 4, "sports": 42, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 17, "sports": 0, "business": 34, "sci_tech": 0, "__failed__": 0 }, "sci_tech": { "world": 7, "sports": 1, "business": 23, "sci_tech": 21, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 91, "accuracy": 0.455, "wilson95": [ 0.387482193098835, 0.524213886275119 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 91, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.455, "brier": 0.6821682024808189, "nll": 1.3997553552410809, "ece10": 0.1487019511508354, "calibration_n": 200, "confident_wrong": 3, "macro_f1": 0.47515787117285196, "confusion_matrix": { "sadness": { "sadness": 30, "joy": 1, "love": 10, "anger": 12, "fear": 3, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 12, "joy": 26, "love": 34, "anger": 0, "fear": 7, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 2, "joy": 2, "love": 8, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 4, "joy": 1, "love": 6, "anger": 14, "fear": 2, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 2, "joy": 0, "love": 4, "anger": 3, "fear": 9, "surprise": 2, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 0, "love": 0, "anger": 0, "fear": 0, "surprise": 4, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 173, "accuracy": 0.865, "wilson95": [ 0.8107086162924745, 0.9055342954521209 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 173, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.865, "brier": 0.35561654122042285, "nll": 0.5423937429604506, "ece10": 0.25880110437456605, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.8631906969673937, "confusion_matrix": { "false": { "false": 98, "true": 11, "__failed__": 0 }, "true": { "false": 16, "true": 75, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 122, "accuracy": 0.61, "wilson95": [ 0.5409374595716976, 0.6749165686253038 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 122, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.4673762480663288, "nll": 0.6591717057756418, "ece10": 0.031665039452511194, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.48, "confusion_matrix": { "true": { "true": 11, "false": 66, "__failed__": 0 }, "false": { "true": 12, "false": 111, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 7, "accuracy": 0.5833333333333334, "wilson95": [ 0.31951131254954973, 0.8067396863412435 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5833333333333334, "brier": 0.6206789502000366, "nll": 1.2502834671273033, "ece10": 0.2949917488850234, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.5866666666666667, "confusion_matrix": { "maintenance": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 1, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 1, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 2, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 2, "accuracy": 0.16666666666666666, "wilson95": [ 0.04696514218385378, 0.4480308622529734 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 2, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.16666666666666666, "brier": 0.8988470569214212, "nll": 1.8756327119155092, "ece10": 0.3482315657322437, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.17333333333333334, "confusion_matrix": { "maintenance": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 1, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 2, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.29547109281361356, "nll": 0.5331028879448334, "ece10": 0.13629193426848463, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8285714285714286, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 2, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 2, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 7, "accuracy": 0.5833333333333334, "wilson95": [ 0.31951131254954973, 0.8067396863412435 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5833333333333334, "brier": 0.5698233562401261, "nll": 0.9737758597559322, "ece10": 0.19083313266296917, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.5833333333333333, "confusion_matrix": { "design": { "design": 2, "inspection": 0, "purchase": 1, "work": 0, "__failed__": 0 }, "inspection": { "design": 1, "inspection": 1, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 1, "purchase": 2, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 1, "purchase": 0, "work": 2, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 74, "accuracy": 0.74, "wilson95": [ 0.6462901055081282, 0.8159530153525186 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 74, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.74, "brier": 0.3297293275152435, "nll": 0.600248032539007, "ece10": 0.038596901615534904, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.7239048880061789, "confusion_matrix": { "label_000": { "label_000": 18, "label_001": 0, "label_002": 1, "label_003": 6, "__failed__": 0 }, "label_001": { "label_000": 9, "label_001": 10, "label_002": 2, "label_003": 4, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 24, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 1, "label_002": 2, "label_003": 22, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.6543337783965972, "nll": 1.3798323295846442, "ece10": 0.09074085675968674, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.4544298880138479, "confusion_matrix": { "label_000": { "label_000": 7, "label_001": 5, "label_002": 2, "label_003": 1, "label_004": 1, "label_005": 1, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 12, "label_002": 0, "label_003": 0, "label_004": 4, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 3, "label_003": 10, "label_004": 4, "label_005": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 15, "label_004": 2, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 2, "label_001": 2, "label_002": 1, "label_003": 4, "label_004": 7, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 2, "label_002": 4, "label_003": 2, "label_004": 3, "label_005": 4, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 44, "accuracy": 0.44, "wilson95": [ 0.34672026630160796, 0.5377189534832302 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 44, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.44, "brier": 0.8178555177566855, "nll": 2.8405394580185925, "ece10": 0.2951466214689427, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.3823302469135803, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 1, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 1, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 1, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 2, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 2, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 1, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 2, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 1, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 1, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 1, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 1, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 1, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 47, "accuracy": 0.9791666666666666, "wilson95": [ 0.8910078200474814, 0.9963128890224893 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 47, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9791666666666666, "brier": 0.07190410630405397, "nll": 0.20036774813942962, "ece10": 0.14547385925083928, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 39, "accuracy": 0.35135135135135137, "wilson95": [ 0.26887011950742146, 0.44377720891493855 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35135135135135137, "brier": 0.8513987585290175, "nll": 1.5920601561164005, "ece10": 0.33406627195139454, "calibration_n": 111, "confident_wrong": 12 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 35, "accuracy": 0.4861111111111111, "wilson95": [ 0.37432674529071863, 0.5993024539480719 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4861111111111111, "cluster_bootstrap95": [ 0.3472222222222222, 0.6388888888888888 ], "independent_source_groups": 36, "brier": 0.7438037711258607, "nll": 1.2005501948593378, "ece10": 0.3383524276109992, "calibration_n": 72, "confident_wrong": 4 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 14, "accuracy": 0.7368421052631579, "wilson95": [ 0.5120844910218489, 0.8819358765321748 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7368421052631579, "brier": 0.2958542718508543, "nll": 0.47601180501243334, "ece10": 0.24762015733368642, "calibration_n": 19, "confident_wrong": 1, "macro_f1": 0.7338935574229692, "confusion_matrix": { "false": { "false": 6, "true": 5, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 6, "accuracy": 0.375, "wilson95": [ 0.18481232558863633, 0.6135895945449727 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.375, "brier": 0.5498586230420717, "nll": 0.7460159503784661, "ece10": 0.22783502675480052, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.2727272727272727, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 8, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 15, "accuracy": 0.6818181818181818, "wilson95": [ 0.47318597741042745, 0.8363940681864375 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6818181818181818, "brier": 0.468734817544779, "nll": 0.6613766769017709, "ece10": 0.13174750253001877, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.6178660049627791, "confusion_matrix": { "false": { "false": 3, "true": 7, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.07523401699601942, "nll": 0.2326049029489122, "ece10": 0.19935241478324556, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.6098628750075317, "nll": 1.2130036194036349, "ece10": 0.3182262642950757, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6314285714285715, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 2, "3": 1, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 16, "accuracy": 0.8421052631578947, "wilson95": [ 0.6243452472967093, 0.9447952836146585 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8421052631578947, "brier": 0.2364879061535118, "nll": 0.5041921859297017, "ece10": 0.24325002002300145, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8092063492063492, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 1, "roads": 0, "noise": 0, "permits": 3, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 2, "noise": 0, "permits": 0, "utilities": 1, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.4389741424796709, "nll": 0.8993576678972781, "ece10": 0.22253969566935067, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7194444444444444, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 1, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 2, "refactor": 1, "nit": 1, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.3615004257774925, "nll": 0.7391565551660827, "ece10": 0.26271176365533183, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8539682539682539, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 1, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 2, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 1, "docs": 0, "test": 0, "chore": 3, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.4683281284496243, "nll": 0.6594681558903414, "ece10": 0.07305623585845153, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.43333333333333335, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.4625536569280615, "nll": 0.6946998489030242, "ece10": 0.2026147008909771, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.6190476190476191, "confusion_matrix": { "false": { "false": 4, "true": 4, "__failed__": 0 }, "true": { "false": 2, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 11, "accuracy": 0.6111111111111112, "wilson95": [ 0.3861904160223791, 0.7969475342783638 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6111111111111112, "brier": 0.5555940756781271, "nll": 1.082810708197936, "ece10": 0.23065199527804614, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.4854545454545455, "confusion_matrix": { "harassment": { "harassment": 2, "threat": 0, "self_harm": 1, "spam": 1, "none": 1, "__failed__": 0 }, "threat": { "harassment": 1, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 1, "none": 4, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 18, "accuracy": 0.8181818181818182, "wilson95": [ 0.6148339256287345, 0.9269311541657794 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8181818181818182, "brier": 0.3727298227211786, "nll": 0.7677627056456928, "ece10": 0.29691172720638015, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8186080586080587, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 1, "opinion": 4, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 1, "opinion": 0, "press_release": 0, "analysis": 3, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 1, "opinion": 1, "press_release": 0, "analysis": 0, "feature": 2, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.17744893874126277, "nll": 0.40239201828284804, "ece10": 0.23242781830983572, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9314285714285715, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 1, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 2, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.09531672640744758, "nll": 0.2686082512911638, "ece10": 0.16867370503544538, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "address_issue": { "address_issue": 3, "customs_hold": 1, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.5238480669579244, "nll": 0.7152540642547315, "ece10": 0.2233323085850764, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.32, "confusion_matrix": { "false": { "false": 0, "true": 9, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.204029010648264, "nll": 0.45302671770028263, "ece10": 0.2390152224616038, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9314285714285715, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "purchase_order": 0, "receipt": 2, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.2492480874739741, "nll": 0.49978111195448954, "ece10": 0.26416802649282295, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8628571428571428, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 2, "thanks": 0, "fyi": 1, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.2636360895098344, "nll": 0.506306518381566, "ece10": 0.2136680717561481, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7825396825396825, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 1, "software": 3, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 1, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 9, "accuracy": 0.5, "wilson95": [ 0.29031021505169974, 0.7096897849483003 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6171376438661604, "nll": 0.819195410711457, "ece10": 0.4069143429905058, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.3333333333333333, "confusion_matrix": { "false": { "false": 0, "true": 9, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.6254583710034721, "nll": 1.1273577656313076, "ece10": 0.3562003834877998, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.625, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 4, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.5773634319234738, "nll": 0.9667137487536783, "ece10": 0.26798642705654263, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6481481481481483, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 4, "1": 2, "2": 1, "__failed__": 0 }, "2": { "0": 1, "1": 0, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 15, "accuracy": 0.75, "wilson95": [ 0.531299122381256, 0.8881382985923343 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.438399727596806, "nll": 0.8614553175088219, "ece10": 0.21203446025205114, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7595238095238095, "confusion_matrix": { "cheating": { "cheating": 2, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 2, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 1, "other": 0, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 3, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 1, "other": 4, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 4, "accuracy": 0.19047619047619047, "wilson95": [ 0.07667565052761793, 0.40000564690204926 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.19047619047619047, "brier": 0.8524231945522202, "nll": 1.6734068418945405, "ece10": 0.22733050009038372, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.064, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 4, "punctuation": 0, "none": 0, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 3, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 4, "punctuation": 0, "none": 0, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 5, "punctuation": 0, "none": 0, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 5, "punctuation": 0, "none": 0, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.602889513779707, "nll": 0.8236081390129316, "ece10": 0.21054535763015328, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.36, "confusion_matrix": { "false": { "false": 0, "true": 7, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 19, "accuracy": 0.8260869565217391, "wilson95": [ 0.6286235235977385, 0.9302134643693744 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8260869565217391, "brier": 0.25101055103640474, "nll": 0.5481134819567391, "ece10": 0.20627326407101723, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 0.8080808080808082, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 4, "hvac": 0, "handyman": 1, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 2, "handyman": 0, "appliance_repair": 2, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 1, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 3, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 8, "accuracy": 0.5714285714285714, "wilson95": [ 0.3259064457965074, 0.7861920109552588 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5714285714285714, "brier": 0.6644717572594042, "nll": 1.3012976640463336, "ece10": 0.2526926868616316, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.5633333333333332, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 1, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.5327256954931163, "nll": 0.9738990797421277, "ece10": 0.15115646221028828, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.46515151515151515, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 0, "2": 2, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 2, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 7, "accuracy": 0.4375, "wilson95": [ 0.23098652405492354, 0.6682144360118811 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4375, "brier": 0.6495544840350206, "nll": 1.1829694729679099, "ece10": 0.17691421673193725, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.35396825396825393, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 0, "3": 2, "__failed__": 0 }, "1": { "0": 3, "1": 0, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 3, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.562724901872597, "nll": 0.7611289281710745, "ece10": 0.09043865021319472, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.34615384615384615, "confusion_matrix": { "false": { "false": 0, "true": 8, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.23930606318127237, "nll": 0.5809146022722592, "ece10": 0.2329609771702501, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.8806349206349205, "confusion_matrix": { "politics": { "politics": 3, "business": 0, "technology": 0, "sports": 0, "culture": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "sports": 2, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.27328597743688887, "nll": 0.57647713232912, "ece10": 0.269014427654911, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8920634920634921, "confusion_matrix": { "payments": { "payments": 3, "web": 0, "infra": 0, "data": 1, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 1, "infra": 3, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.47935711477172943, "nll": 0.6681708204373766, "ece10": 0.24777293977930023, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.4711111111111111, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.7778239543004062, "nll": 1.1057137656799285, "ece10": 0.3951063031739143, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.3333333333333333, "confusion_matrix": { "false": { "false": 0, "true": 8, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 7, "accuracy": 0.4375, "wilson95": [ 0.23098652405492354, 0.6682144360118811 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4375, "brier": 0.5969133216150042, "nll": 1.0150956086732577, "ece10": 0.19457471907293766, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.3746498599439776, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "2": { "0": 3, "1": 2, "2": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.331953037578381, "nll": 0.7499110906711358, "ece10": 0.1922205849307234, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7678571428571428, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 1, "french": 1, "mexican": 0, "indian": 2, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 1, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 6, "accuracy": 0.35294117647058826, "wilson95": [ 0.17309720397243744, 0.5869963651478132 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35294117647058826, "brier": 0.7709162010227539, "nll": 1.1099985049556316, "ece10": 0.3886374608981078, "calibration_n": 17, "confident_wrong": 3, "macro_f1": 0.2608695652173913, "confusion_matrix": { "false": { "false": 0, "true": 8, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 10, "accuracy": 0.5555555555555556, "wilson95": [ 0.33716415642042774, 0.7544048187299437 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5555555555555556, "brier": 0.3983339743094742, "nll": 0.7444799585959001, "ece10": 0.2734023905562208, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5523809523809524, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 4, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 2, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 2, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 10, "accuracy": 0.5555555555555556, "wilson95": [ 0.33716415642042774, 0.7544048187299437 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5555555555555556, "brier": 0.6458879281312135, "nll": 1.2549381283368415, "ece10": 0.1979300736335949, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5164102564102564, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 2, "1": 0, "2": 1, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 1, "4": 2, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.5939820457389785, "nll": 0.8455757180212626, "ece10": 0.305835115019846, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6190476190476191, "confusion_matrix": { "false": { "false": 4, "true": 6, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 7, "accuracy": 0.4666666666666667, "wilson95": [ 0.24809535365737387, 0.698830199745016 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.8873780066247883, "nll": 1.327643933890305, "ece10": 0.44220185800401995, "calibration_n": 15, "confident_wrong": 4, "macro_f1": 0.3181818181818182, "confusion_matrix": { "false": { "false": 0, "true": 8, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.31525392943185665, "nll": 0.6413712911063775, "ece10": 0.3938114629751861, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 4, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.567194945986918, "nll": 0.766849610091139, "ece10": 0.29201754653307777, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.34615384615384615, "confusion_matrix": { "false": { "false": 0, "true": 8, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 7, "accuracy": 0.3684210526315789, "wilson95": [ 0.19149512745135072, 0.5896046683519693 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3684210526315789, "brier": 0.6969811661752506, "nll": 1.2746541059039553, "ece10": 0.08880499384835615, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.20535714285714285, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 4, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 2, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 2, "1": 1, "2": 2, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 6, "accuracy": 0.35294117647058826, "wilson95": [ 0.17309720397243744, 0.5869963651478132 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35294117647058826, "brier": 0.7360006776679303, "nll": 0.9581092510661183, "ece10": 0.39941317128419146, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.2608695652173913, "confusion_matrix": { "false": { "false": 0, "true": 11, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.08882120948949879, "nll": 0.21508316271283948, "ece10": 0.18733460550817863, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 12, "accuracy": 0.6, "wilson95": [ 0.3865815007622531, 0.781193467627183 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.6555201869620415, "nll": 1.205970609681779, "ece10": 0.2801859929684969, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.5128205128205128, "confusion_matrix": { "0": { "0": 5, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 1, "1": 0, "2": 4, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 18, "accuracy": 0.9, "wilson95": [ 0.6989663547715128, 0.9721335187862319 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.31825210842527574, "nll": 0.6740241306697896, "ece10": 0.3530734184171098, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9099999999999999, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 3, "reminder": 0, "music": 0, "small_talk": 2, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 15, "accuracy": 0.75, "wilson95": [ 0.531299122381256, 0.8881382985923343 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.4076578992047429, "nll": 0.5958981107267667, "ece10": 0.15259482973065314, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7493734335839599, "confusion_matrix": { "false": { "false": 8, "true": 1, "__failed__": 0 }, "true": { "false": 4, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 7, "accuracy": 0.3181818181818182, "wilson95": [ 0.1636059318135624, 0.5268140225895723 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3181818181818182, "brier": 0.7409300754074107, "nll": 1.381836720346803, "ece10": 0.104988482976952, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.18055555555555555, "confusion_matrix": { "0": { "0": 0, "1": 0, "2": 2, "3": 3, "__failed__": 0 }, "1": { "0": 0, "1": 0, "2": 1, "3": 5, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 472, "accuracy": 0.5244444444444445, "wilson95": [ 0.4917827897252107, 0.5568983142124311 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 593, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5526561043802423, "cluster_bootstrap95": [ 0.49227594728358604, 0.5550240369338042 ], "independent_source_groups": 596, "brier": 0.5322905840365911, "nll": 0.8948278038539144, "ece10": 0.037367119052093875, "calibration_n": 1073, "confident_wrong": 0 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 242, "accuracy": 0.42160278745644597, "wilson95": [ 0.38185736835724604, 0.46239056742832507 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 541, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5779914529914529, "brier": 0.5746489113288867, "nll": 1.1836648565906174, "ece10": 0.051249955989428345, "calibration_n": 936, "confident_wrong": 14 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 135, "accuracy": 0.19285714285714287, "wilson95": [ 0.16533765374640866, 0.22372930925904308 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 317, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2913602941176471, "cluster_bootstrap95": [ 0.1635208506532828, 0.22286511342734516 ], "independent_source_groups": 650, "brier": 0.8177142227782064, "nll": 1.553057546389038, "ece10": 0.23739554048425565, "calibration_n": 1088, "confident_wrong": 19 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 13, "accuracy": 0.0325, "wilson95": [ 0.01908984237880045, 0.0548041528237668 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 915, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4575, "brier": 0.658396176549997, "nll": 1.131749418328986, "ece10": 0.11106102086862973, "calibration_n": 2000, "confident_wrong": 2 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "wilson95": [ 0.7735463743160095, 0.9801879358846942 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 26, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.18036374677438466, "nll": 0.39652788803801986, "ece10": 0.21198840980728936, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.925, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 1, "inspection": 5, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.20632235345686797, "nll": 0.43549119476565246, "ece10": 0.27352501319127104, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 0, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 21, "accuracy": 0.75, "wilson95": [ 0.5664432968100104, 0.8732350508070668 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.4389714559895799, "nll": 0.7983623457987677, "ece10": 0.20992050361716383, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.7636639676113361, "confusion_matrix": { "design": { "design": 6, "inspection": 1, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 3, "inspection": 4, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 2, "inspection": 0, "purchase": 5, "work": 0, "__failed__": 0 }, "work": { "design": 1, "inspection": 0, "purchase": 0, "work": 6, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 22, "accuracy": 0.7857142857142857, "wilson95": [ 0.6046141434627812, 0.8978753966710211 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.4869109336902847, "nll": 0.8916689286047602, "ece10": 0.29944437605574387, "calibration_n": 28, "confident_wrong": 1, "macro_f1": 0.7948717948717948, "confusion_matrix": { "work": { "work": 6, "purchase": 0, "inspection": 0, "design": 1, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 6, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 4, "design": 3, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 1, "design": 6, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 31, "accuracy": 0.775, "wilson95": [ 0.6249690325768998, 0.8768390867645927 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 31, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.775, "brier": 0.34112738053660274, "nll": 0.6931050303758581, "ece10": 0.16073375468710416, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7438383838383839, "confusion_matrix": { "review": { "review": 2, "it": 3, "documents": 1, "procurement": 1, "maintenance": 1, "__failed__": 0 }, "it": { "review": 0, "it": 7, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 1, "it": 0, "documents": 0, "procurement": 1, "maintenance": 6, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 32, "accuracy": 0.8, "wilson95": [ 0.652426936536005, 0.895000102745623 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 32, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.33488727076874175, "nll": 0.6901220585512264, "ece10": 0.17744147185178216, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7661111111111111, "confusion_matrix": { "maintenance": { "maintenance": 7, "procurement": 1, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 7, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 1, "documents": 1, "it": 3, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 22, "accuracy": 0.55, "wilson95": [ 0.39829091798932054, 0.6929469218909508 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.568472061799832, "nll": 1.1174253614608818, "ece10": 0.09600472778160085, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.5532828282828283, "confusion_matrix": { "review": { "review": 2, "it": 2, "documents": 0, "procurement": 4, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 7, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 5, "procurement": 3, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 1, "it": 1, "documents": 0, "procurement": 3, "maintenance": 3, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 3, "maintenance": 5, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 23, "accuracy": 0.575, "wilson95": [ 0.42195068541991065, 0.7149060744004962 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.575, "brier": 0.5873182450859216, "nll": 1.126816386193934, "ece10": 0.11608064399908001, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.58, "confusion_matrix": { "maintenance": { "maintenance": 5, "procurement": 3, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 2, "procurement": 5, "documents": 0, "it": 1, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 3, "documents": 5, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 6, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 5, "documents": 0, "it": 1, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 196, "accuracy": 0.49, "wilson95": [ 0.441339288394901, 0.5388509574917849 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 196, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.6744917538673234, "nll": 1.405774830729498, "ece10": 0.09480279732965095, "calibration_n": 400, "confident_wrong": 8, "macro_f1": 0.45367453522568635, "confusion_matrix": { "sadness": { "sadness": 67, "joy": 1, "love": 26, "anger": 17, "fear": 7, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 15, "joy": 38, "love": 48, "anger": 10, "fear": 5, "surprise": 2, "__failed__": 0 }, "love": { "sadness": 6, "joy": 3, "love": 24, "anger": 2, "fear": 2, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 7, "joy": 5, "love": 13, "anger": 34, "fear": 2, "surprise": 3, "__failed__": 0 }, "fear": { "sadness": 4, "joy": 2, "love": 3, "anger": 7, "fear": 30, "surprise": 6, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 2, "love": 1, "anger": 0, "fear": 2, "surprise": 3, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 232, "accuracy": 0.7733333333333333, "wilson95": [ 0.7226746795338672, 0.8170804961967142 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 232, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7733333333333333, "brier": 0.4323591064855096, "nll": 1.1020695480302456, "ece10": 0.27252510272192504, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 116, "accuracy": 0.29, "wilson95": [ 0.24769665928572088, 0.3362985043346836 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 116, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.29, "brier": 0.6658343322229618, "nll": 0.8695931924860055, "ece10": 0.40723313333354283, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.27962662337662336, "confusion_matrix": { "false": { "false": 34, "true": 275, "__failed__": 0 }, "true": { "false": 9, "true": 82, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 285, "accuracy": 0.7125, "wilson95": [ 0.6662901779980862, 0.7546670969098378 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 285, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7125, "brier": 0.42759809726988457, "nll": 0.8391203302763965, "ece10": 0.1277544885350312, "calibration_n": 400, "confident_wrong": 29, "macro_f1": 0.6784310564852162, "confusion_matrix": { "world": { "world": 89, "sports": 6, "business": 7, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 8, "sports": 113, "business": 2, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 20, "sports": 2, "business": 49, "sci_tech": 1, "__failed__": 0 }, "sci_tech": { "world": 29, "sports": 0, "business": 39, "sci_tech": 34, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 233, "accuracy": 0.5825, "wilson95": [ 0.5336117391164401, 0.629818732318401 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 233, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5825, "brier": 0.48483960593873937, "nll": 0.6781227647515103, "ece10": 0.03598390251336794, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.4297957337111933, "confusion_matrix": { "true": { "true": 13, "false": 140, "__failed__": 0 }, "false": { "true": 27, "false": 220, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 155, "accuracy": 0.5166666666666667, "wilson95": [ 0.4602663137927717, 0.5726455871664101 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 155, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5166666666666667, "brier": 0.630076612802019, "nll": 1.571563742613362, "ece10": 0.1135288150945125, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 339, "accuracy": 0.8475, "wilson95": [ 0.8089761434535727, 0.8794128119840912 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 339, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8475, "brier": 0.35776550978693933, "nll": 0.5449782001527947, "ece10": 0.24179210317801758, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.845717067865521, "confusion_matrix": { "false": { "false": 148, "true": 40, "__failed__": 0 }, "true": { "false": 21, "true": 191, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 206, "accuracy": 0.515, "wilson95": [ 0.46611356033171036, 0.5636010708382608 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 206, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.515, "brier": 0.5367884919901401, "nll": 0.7305196239991186, "ece10": 0.19711713932761277, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.38837920489296635, "confusion_matrix": { "false": { "false": 12, "true": 188, "__failed__": 0 }, "true": { "false": 6, "true": 194, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 77, "accuracy": 0.77, "wilson95": [ 0.6784561697712622, 0.8415673411969654 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 77, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.77, "brier": 0.31227229562034203, "nll": 0.48618401483806695, "ece10": 0.05071808062128602, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.7681217864704104, "confusion_matrix": { "yes": { "yes": 43, "no": 11, "__failed__": 0 }, "no": { "yes": 12, "no": 34, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.5291281801821915, "nll": 1.2010768224370723, "ece10": 0.09935486285894397, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5729734964109965, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 5, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 3, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 8, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 1, "card_lost": 1, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "statement": { "balance": 1, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 4, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 2, "branch_appointment": 2, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 5, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 2, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 7, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 1, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 4, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 4, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 5, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 1, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 59, "accuracy": 0.59, "wilson95": [ 0.4920143005341078, 0.6813268697886348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 156, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.78, "brier": 0.30689923970877747, "nll": 0.5155024465227581, "ece10": 0.060536987025250094, "calibration_n": 200, "confident_wrong": 2 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 45, "accuracy": 0.45, "wilson95": [ 0.3561453797951198, 0.5475539700255787 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 130, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.47707374506170785, "nll": 0.9623160563104902, "ece10": 0.06975408267391175, "calibration_n": 200, "confident_wrong": 5 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 78, "accuracy": 0.78, "wilson95": [ 0.6892964648501421, 0.8499871761539459 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 78, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.78, "brier": 0.3632415737806332, "nll": 0.9290747213137456, "ece10": 0.18570296785866677, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.7441964285714285, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 13, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 5, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 3, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 0, "receipt": 2, "contract": 0, "resume": 0, "bank_statement": 13, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 6, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 2, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 9, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 6, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 17, "accuracy": 0.17, "wilson95": [ 0.10893566114059931, 0.25548004767601123 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 246, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.615, "brier": 0.5209460697756727, "nll": 0.9674082032551456, "ece10": 0.05645169882069038, "calibration_n": 400, "confident_wrong": 1 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 63, "accuracy": 0.63, "wilson95": [ 0.5322052958094291, 0.7181763946567546 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 63, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.63, "brier": 0.510696184994647, "nll": 1.0847908859642872, "ece10": 0.08300002048853693, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6311604803864866, "confusion_matrix": { "politics": { "politics": 2, "business": 1, "technology": 2, "science": 5, "health": 2, "sports": 0, "entertainment": 0, "world": 0, "culture": 1, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 10, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 1, "technology": 7, "science": 2, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 1, "business": 1, "technology": 0, "science": 7, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "science": 2, "health": 1, "sports": 4, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 0, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 7, "world": 0, "culture": 1, "weather": 0, "__failed__": 0 }, "world": { "politics": 2, "business": 2, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 3, "culture": 1, "weather": 0, "__failed__": 0 }, "culture": { "politics": 1, "business": 0, "technology": 0, "science": 1, "health": 1, "sports": 0, "entertainment": 0, "world": 1, "culture": 6, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 2, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 7, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 46, "accuracy": 0.46, "wilson95": [ 0.36560808309024106, 0.5573513967663177 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 46, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.46, "brier": 0.6808224615460832, "nll": 1.5632173998119914, "ece10": 0.09963095330299304, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.44770510993421214, "confusion_matrix": { "biology": { "biology": 4, "medicine": 2, "chemistry": 1, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 5, "chemistry": 0, "physics": 0, "computer_science": 2, "mathematics": 1, "earth_science": 0, "engineering": 0, "economics": 1, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 6, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 2, "medicine": 1, "chemistry": 1, "physics": 9, "computer_science": 0, "mathematics": 1, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 1, "medicine": 2, "chemistry": 2, "physics": 1, "computer_science": 5, "mathematics": 2, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 4, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 2, "computer_science": 2, "mathematics": 0, "earth_science": 4, "engineering": 0, "economics": 1, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 0, "chemistry": 2, "physics": 0, "computer_science": 1, "mathematics": 2, "earth_science": 0, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 6, "other": 0, "__failed__": 0 }, "other": { "biology": 3, "medicine": 2, "chemistry": 0, "physics": 2, "computer_science": 0, "mathematics": 1, "earth_science": 1, "engineering": 1, "economics": 1, "other": 1, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.3943512277018881, "nll": 0.8025973761422585, "ece10": 0.12352311028476265, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 6, "accuracy": 0.06, "wilson95": [ 0.027786123963188192, 0.1247681544589592 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 92, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.46, "brier": 0.2762980889758956, "nll": 0.49522586994521345, "ece10": 0.10784615307827308, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 82, "accuracy": 0.82, "wilson95": [ 0.7333264161369926, 0.8829977450105364 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 82, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.82, "brier": 0.25204870016844205, "nll": 0.44952996923780913, "ece10": 0.0934931037323824, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8148557796573411, "confusion_matrix": { "negative": { "negative": 36, "neutral": 0, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 12, "neutral": 19, "positive": 2, "__failed__": 0 }, "positive": { "negative": 3, "neutral": 1, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 73, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.365, "brier": 0.3117679358362806, "nll": 0.47433445651601425, "ece10": 0.12924122468716323, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 33, "accuracy": 0.33, "wilson95": [ 0.24563122734128034, 0.4269465620490947 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 208, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6933333333333334, "brier": 0.39375792805585214, "nll": 0.7746272263871998, "ece10": 0.038635837628475826, "calibration_n": 300, "confident_wrong": 3 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 69, "accuracy": 0.69, "wilson95": [ 0.5937394428051057, 0.7722030278762395 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.69, "brier": 0.45986257460996305, "nll": 1.2083772220405513, "ece10": 0.1148227467245468, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.6523225782154355, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 2, "password_reset": 1, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 1, "password_reset": 5, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 4, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 2, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 4, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 3, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 4, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 7, "tax_document": 2, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 5, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 6, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 2, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 42, "accuracy": 0.42, "wilson95": [ 0.3279838267435473, 0.5179351329695703 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 42, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42, "brier": 0.750900281234086, "nll": 1.746360608991965, "ece10": 0.12533741946223909, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.4011028459557871, "confusion_matrix": { "billing": { "billing": 6, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 1, "account_access": 7, "shipping": 1, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 1, "account_access": 0, "shipping": 4, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 3, "account_access": 2, "shipping": 1, "returns": 0, "technical_outage": 1, "onboarding": 0, "integrations": 3, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 3, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 5, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "compliance": { "billing": 1, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 2, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 2, "data_privacy": 1, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 5, "other": 0, "__failed__": 0 }, "other": { "billing": 4, "account_access": 0, "shipping": 2, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 1, "data_privacy": 1, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 14, "accuracy": 0.14, "wilson95": [ 0.08526343659939134, 0.22137188210963837 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 162, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.54, "brier": 0.5389935239625467, "nll": 1.1187045571657184, "ece10": 0.043915503339097904, "calibration_n": 300, "confident_wrong": 0 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 61, "accuracy": 0.61, "wilson95": [ 0.5120301704584073, 0.6998312599360558 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 61, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.5096521420171402, "nll": 1.1185858975532812, "ece10": 0.12722799215853545, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5964324618736383, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 4, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 2, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 4, "hotel": 0, "car": 2, "visa_docs": 0, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 9, "car": 0, "visa_docs": 1, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 4, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 6, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 1, "loyalty": 5, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 1, "visa_docs": 0, "loyalty": 1, "disruption": 3, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 4, "car": 2, "visa_docs": 1, "loyalty": 1, "disruption": 0, "other": 2, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 73, "accuracy": 0.73, "wilson95": [ 0.6356788323205499, 0.8073041585042365 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 73, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.73, "brier": 0.37456620750129127, "nll": 0.6704781920741687, "ece10": 0.1188559418010723, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.6886112629852379, "confusion_matrix": { "business": { "business": 21, "science and technology": 0, "sports": 1, "world": 3, "__failed__": 0 }, "science and technology": { "business": 12, "science and technology": 5, "sports": 1, "world": 7, "__failed__": 0 }, "sports": { "business": 1, "science and technology": 0, "sports": 23, "world": 1, "__failed__": 0 }, "world": { "business": 0, "science and technology": 1, "sports": 0, "world": 24, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 76, "accuracy": 0.76, "wilson95": [ 0.6676766365018553, 0.8330867444305122 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 76, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.5098880386389719, "nll": 1.4233236896794907, "ece10": 0.3638887694265407, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.7417328042328042, "confusion_matrix": { "refund not showing up": { "refund not showing up": 1, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 1, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 2, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 1, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 1, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 1, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 2, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 2, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 1, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 49, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.6403362065236654, "nll": 1.334465106752136, "ece10": 0.1008069894753808, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.4673082184857547, "confusion_matrix": { "anger": { "anger": 9, "fear": 2, "joy": 1, "love": 1, "sadness": 3, "surprise": 1, "__failed__": 0 }, "fear": { "anger": 1, "fear": 10, "joy": 0, "love": 1, "sadness": 3, "surprise": 2, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 2, "love": 10, "sadness": 3, "surprise": 1, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 0, "love": 15, "sadness": 2, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 3, "fear": 2, "joy": 1, "love": 4, "sadness": 6, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 2, "joy": 2, "love": 2, "sadness": 2, "surprise": 7, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1729, "accuracy": 0.5864993215739485, "wilson95": [ 0.5686210598294915, 0.6041524468492016 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1729, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5864993215739485, "brier": 0.5873069476304357, "nll": 1.458216565736965, "ece10": 0.1424196257400005, "calibration_n": 2948, "confident_wrong": 4, "macro_f1": 0.6219116582506914, "confusion_matrix": { "alarm": { "alarm": 88, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 7, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 42, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 14, "iot": 0, "lists": 0, "music": 3, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 17, "audio": 0, "calendar": 251, "cooking": 7, "datetime": 15, "email": 1, "general": 85, "iot": 0, "lists": 4, "music": 5, "news": 2, "play": 1, "qa": 0, "recommendation": 2, "social": 3, "takeaway": 2, "transport": 4, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 67, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 26, "cooking": 0, "datetime": 70, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 228, "general": 38, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "general": { "alarm": 0, "audio": 3, "calendar": 2, "cooking": 5, "datetime": 0, "email": 0, "general": 135, "iot": 4, "lists": 2, "music": 4, "news": 2, "play": 4, "qa": 7, "recommendation": 4, "social": 6, "takeaway": 0, "transport": 5, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 37, "datetime": 1, "email": 0, "general": 75, "iot": 98, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 16, "iot": 0, "lists": 116, "music": 3, "news": 0, "play": 2, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 2, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 73, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 18, "iot": 1, "lists": 0, "music": 0, "news": 101, "play": 1, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 93, "calendar": 0, "cooking": 0, "datetime": 1, "email": 1, "general": 27, "iot": 0, "lists": 0, "music": 151, "news": 0, "play": 110, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 1, "calendar": 2, "cooking": 0, "datetime": 2, "email": 0, "general": 242, "iot": 2, "lists": 3, "music": 2, "news": 3, "play": 8, "qa": 14, "recommendation": 0, "social": 2, "takeaway": 0, "transport": 3, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 0, "general": 50, "iot": 1, "lists": 3, "music": 2, "news": 2, "play": 9, "qa": 0, "recommendation": 21, "social": 1, "takeaway": 0, "transport": 2, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 0, "email": 1, "general": 25, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 75, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 5, "datetime": 2, "email": 0, "general": 14, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 27, "transport": 7, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 15, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 104, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 36, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 5, "weather": 109, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 729, "accuracy": 0.2472862957937585, "wilson95": [ 0.232047866064693, 0.26318247716814525 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 729, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2472862957937585, "brier": 0.8742912320698711, "nll": 2.518347882066193, "ece10": 0.07955653940558868, "calibration_n": 2948, "confident_wrong": 0, "macro_f1": 0.2734922118514774, "confusion_matrix": { "alarm": { "alarm": 79, "audio": 0, "calendar": 0, "cooking": 2, "datetime": 0, "email": 0, "general": 10, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "audio": { "alarm": 2, "audio": 1, "calendar": 0, "cooking": 7, "datetime": 3, "email": 0, "general": 34, "iot": 1, "lists": 0, "music": 3, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 4, "weather": 6, "__failed__": 0 }, "calendar": { "alarm": 0, "audio": 0, "calendar": 107, "cooking": 13, "datetime": 12, "email": 1, "general": 159, "iot": 5, "lists": 3, "music": 4, "news": 10, "play": 1, "qa": 1, "recommendation": 6, "social": 1, "takeaway": 0, "transport": 9, "weather": 68, "__failed__": 0 }, "cooking": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 28, "datetime": 1, "email": 0, "general": 18, "iot": 0, "lists": 2, "music": 0, "news": 1, "play": 0, "qa": 5, "recommendation": 1, "social": 0, "takeaway": 2, "transport": 4, "weather": 8, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 8, "datetime": 6, "email": 0, "general": 66, "iot": 0, "lists": 0, "music": 0, "news": 2, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 4, "weather": 11, "__failed__": 0 }, "email": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 3, "datetime": 1, "email": 63, "general": 111, "iot": 1, "lists": 3, "music": 0, "news": 7, "play": 0, "qa": 0, "recommendation": 6, "social": 61, "takeaway": 1, "transport": 3, "weather": 7, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 10, "datetime": 1, "email": 4, "general": 107, "iot": 3, "lists": 2, "music": 1, "news": 1, "play": 4, "qa": 0, "recommendation": 10, "social": 0, "takeaway": 2, "transport": 15, "weather": 25, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 7, "calendar": 0, "cooking": 52, "datetime": 0, "email": 0, "general": 112, "iot": 15, "lists": 1, "music": 2, "news": 0, "play": 0, "qa": 0, "recommendation": 7, "social": 1, "takeaway": 0, "transport": 8, "weather": 8, "__failed__": 0 }, "lists": { "alarm": 1, "audio": 27, "calendar": 1, "cooking": 5, "datetime": 0, "email": 0, "general": 23, "iot": 0, "lists": 79, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 3, "__failed__": 0 }, "music": { "alarm": 0, "audio": 9, "calendar": 0, "cooking": 2, "datetime": 0, "email": 0, "general": 31, "iot": 1, "lists": 1, "music": 26, "news": 2, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 2, "transport": 2, "weather": 3, "__failed__": 0 }, "news": { "alarm": 1, "audio": 0, "calendar": 1, "cooking": 2, "datetime": 1, "email": 0, "general": 70, "iot": 0, "lists": 1, "music": 0, "news": 29, "play": 0, "qa": 0, "recommendation": 6, "social": 1, "takeaway": 1, "transport": 3, "weather": 8, "__failed__": 0 }, "play": { "alarm": 3, "audio": 104, "calendar": 1, "cooking": 15, "datetime": 0, "email": 0, "general": 137, "iot": 2, "lists": 19, "music": 79, "news": 3, "play": 4, "qa": 0, "recommendation": 4, "social": 1, "takeaway": 3, "transport": 3, "weather": 7, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 18, "datetime": 0, "email": 0, "general": 185, "iot": 8, "lists": 2, "music": 5, "news": 3, "play": 4, "qa": 2, "recommendation": 7, "social": 0, "takeaway": 3, "transport": 18, "weather": 29, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 2, "calendar": 0, "cooking": 8, "datetime": 0, "email": 0, "general": 40, "iot": 2, "lists": 0, "music": 8, "news": 0, "play": 9, "qa": 0, "recommendation": 7, "social": 0, "takeaway": 0, "transport": 8, "weather": 10, "__failed__": 0 }, "social": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 2, "datetime": 0, "email": 1, "general": 28, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 65, "takeaway": 1, "transport": 2, "weather": 2, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 14, "datetime": 0, "email": 0, "general": 20, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 4, "social": 0, "takeaway": 12, "transport": 4, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 4, "datetime": 0, "email": 8, "general": 43, "iot": 1, "lists": 3, "music": 0, "news": 3, "play": 0, "qa": 0, "recommendation": 7, "social": 0, "takeaway": 0, "transport": 46, "weather": 4, "__failed__": 0 }, "weather": { "alarm": 1, "audio": 0, "calendar": 4, "cooking": 3, "datetime": 0, "email": 0, "general": 70, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 18, "weather": 53, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 484, "accuracy": 0.41261722080136404, "wilson95": [ 0.3847740118264303, 0.44103090143229423 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 484, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.41261722080136404, "cluster_bootstrap95": [ 0.37731466986530865, 0.4479045394059536 ], "independent_source_groups": 47, "brier": 0.6786142542499621, "nll": 1.1273260331964086, "ece10": 0.13689967518144333, "calibration_n": 1173, "confident_wrong": 0, "macro_f1": 0.3763498070958944, "confusion_matrix": { "negative": { "negative": 98, "neutral": 61, "positive": 23, "__failed__": 0 }, "neutral": { "negative": 225, "neutral": 306, "positive": 59, "__failed__": 0 }, "positive": { "negative": 193, "neutral": 128, "positive": 80, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "gliner-decide-multi": { "label": "GLiNER Decide multilingual", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 149, "accuracy": 0.745, "wilson95": [ 0.6803707417954529, 0.8003950482796865 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 149, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.745, "brier": 0.3630818454161166, "nll": 0.6916825595302629, "ece10": 0.03726413847824407, "calibration_n": 200, "confident_wrong": 4, "macro_f1": 0.7452612159631964, "confusion_matrix": { "world": { "world": 42, "sports": 4, "business": 2, "sci_tech": 2, "__failed__": 0 }, "sports": { "world": 2, "sports": 44, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 15, "sports": 0, "business": 32, "sci_tech": 4, "__failed__": 0 }, "sci_tech": { "world": 7, "sports": 2, "business": 12, "sci_tech": 31, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 78, "accuracy": 0.39, "wilson95": [ 0.3250834313746962, 0.4590625404283025 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 78, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.39, "brier": 0.714173566130998, "nll": 1.475754936417346, "ece10": 0.11010146317841002, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.31923349191886513, "confusion_matrix": { "sadness": { "sadness": 11, "joy": 8, "love": 5, "anger": 1, "fear": 32, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 2, "joy": 45, "love": 17, "anger": 0, "fear": 14, "surprise": 1, "__failed__": 0 }, "love": { "sadness": 0, "joy": 6, "love": 5, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 0, "joy": 3, "love": 5, "anger": 4, "fear": 16, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 1, "joy": 2, "love": 4, "anger": 0, "fear": 12, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 125, "accuracy": 0.625, "wilson95": [ 0.5561430369897816, 0.6891456314159018 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 125, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.4761102028371876, "nll": 0.6691774438409722, "ece10": 0.053566888917693005, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.5836918209319752, "confusion_matrix": { "false": { "false": 94, "true": 15, "__failed__": 0 }, "true": { "false": 60, "true": 31, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 129, "accuracy": 0.645, "wilson95": [ 0.5765211307549043, 0.7080137245956883 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 129, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.645, "brier": 0.42750686118922276, "nll": 0.6164205440001321, "ece10": 0.06148736858940007, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.5523187994577383, "confusion_matrix": { "true": { "true": 19, "false": 58, "__failed__": 0 }, "false": { "true": 13, "false": 110, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 5, "accuracy": 0.4166666666666667, "wilson95": [ 0.19326031365875665, 0.6804886874504503 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4166666666666667, "brier": 0.6741989766518933, "nll": 1.325533348513371, "ece10": 0.18589305966863648, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.43999999999999995, "confusion_matrix": { "maintenance": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 1, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 0, "documents": 1, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 3, "accuracy": 0.25, "wilson95": [ 0.08894166839405471, 0.5323053349335657 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 3, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.25, "brier": 0.8299449315040371, "nll": 1.5515334487533539, "ece10": 0.31484777588615204, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.24000000000000005, "confusion_matrix": { "maintenance": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 1, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 2, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 3, "accuracy": 0.25, "wilson95": [ 0.08894166839405471, 0.5323053349335657 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 3, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.25, "brier": 0.6878990983608775, "nll": 1.110732370335878, "ece10": 0.33452047372404714, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.22916666666666663, "confusion_matrix": { "design": { "design": 1, "inspection": 0, "purchase": 2, "work": 0, "__failed__": 0 }, "inspection": { "design": 1, "inspection": 1, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 1, "purchase": 1, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 6, "accuracy": 0.5, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.7508295418085066, "nll": 1.3551721676384112, "ece10": 0.2391308271786112, "calibration_n": 12, "confident_wrong": 1, "macro_f1": 0.4095238095238095, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 1, "inspection": 1, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 1, "purchase": 0, "work": 1, "__failed__": 0 }, "work": { "design": 1, "inspection": 0, "purchase": 0, "work": 2, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 78, "accuracy": 0.78, "wilson95": [ 0.6892964648501421, 0.8499871761539459 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 78, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.78, "brier": 0.36343620861382353, "nll": 0.7133837996613832, "ece10": 0.19856706563080664, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.7688676347732075, "confusion_matrix": { "label_000": { "label_000": 20, "label_001": 0, "label_002": 1, "label_003": 4, "__failed__": 0 }, "label_001": { "label_000": 9, "label_001": 11, "label_002": 1, "label_003": 4, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 24, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 23, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 49, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.6802847049145265, "nll": 1.415172915388432, "ece10": 0.14931041926250227, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.4580952078263291, "confusion_matrix": { "label_000": { "label_000": 5, "label_001": 6, "label_002": 2, "label_003": 1, "label_004": 3, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 15, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 11, "label_003": 5, "label_004": 1, "label_005": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 2, "label_002": 3, "label_003": 11, "label_004": 1, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 1, "label_001": 7, "label_002": 4, "label_003": 1, "label_004": 3, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 1, "label_002": 7, "label_003": 3, "label_004": 1, "label_005": 4, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 61, "accuracy": 0.61, "wilson95": [ 0.5120301704584073, 0.6998312599360558 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 61, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.7111796418442353, "nll": 2.2074070007276263, "ece10": 0.3872811547690316, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5880291005291005, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 1, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 1, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 1, "label_007": 1, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 1, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 1, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 1, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 1, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 1, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 1, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 1, "label_026": 1, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 1, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 1, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 1, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 1, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 1, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 1, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 47, "accuracy": 0.9791666666666666, "wilson95": [ 0.8910078200474814, 0.9963128890224893 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 47, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9791666666666666, "brier": 0.08214570580481437, "nll": 0.2021952556824608, "ece10": 0.14632390634540687, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 42, "accuracy": 0.3783783783783784, "wilson95": [ 0.2936531154974464, 0.47124015321175733 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 42, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3783783783783784, "brier": 0.7746963432769919, "nll": 1.3979011497278397, "ece10": 0.22234485714570107, "calibration_n": 111, "confident_wrong": 3 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 42, "accuracy": 0.5833333333333334, "wilson95": [ 0.46807697931465553, 0.6901478252526017 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 42, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5833333333333334, "cluster_bootstrap95": [ 0.4440972222222222, 0.7361111111111112 ], "independent_source_groups": 36, "brier": 0.6320763499539533, "nll": 1.076651141498099, "ece10": 0.2568314363566274, "calibration_n": 72, "confident_wrong": 3 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 13, "accuracy": 0.6842105263157895, "wilson95": [ 0.46010407675327475, 0.8463562091220773 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6842105263157895, "brier": 0.3951874754129674, "nll": 0.5738855284526254, "ece10": 0.0659348188280024, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.6833333333333333, "confusion_matrix": { "false": { "false": 6, "true": 5, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.39775539101638674, "nll": 0.5867576155135467, "ece10": 0.14423823467269603, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7333333333333334, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 12, "accuracy": 0.5454545454545454, "wilson95": [ 0.3465979532073494, 0.7307970581918668 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5454545454545454, "brier": 0.5596747776799629, "nll": 0.7582747077898412, "ece10": 0.16717240418541335, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.35294117647058826, "confusion_matrix": { "false": { "false": 0, "true": 10, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.23301481822044845, "nll": 0.4909357811698578, "ece10": 0.25896022804697355, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.886904761904762, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 3, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 1, "feature_request": 0, "praise": 0, "pricing_billing": 3, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 4, "accuracy": 0.2222222222222222, "wilson95": [ 0.09000928108601694, 0.45214584316212625 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2222222222222222, "brier": 0.7627776130813377, "nll": 1.5154129888984036, "ece10": 0.3219962326005896, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.07272727272727272, "confusion_matrix": { "0": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 0, "2": 3, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 3, "3": 0, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.37667881727875335, "nll": 0.7942066889921342, "ece10": 0.4381067284562644, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9377777777777778, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 1, "roads": 0, "noise": 0, "permits": 0, "utilities": 2, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.6840304379963704, "nll": 1.4627664054296372, "ece10": 0.11958914836023507, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.3029100529100529, "confusion_matrix": { "bug": { "bug": 1, "performance": 0, "security": 0, "style": 2, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 0, "security": 0, "style": 2, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 1, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 4, "refactor": 0, "nit": 0, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 7, "accuracy": 0.4666666666666667, "wilson95": [ 0.24809535365737387, 0.698830199745016 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.6944311504687395, "nll": 1.4468650383787407, "ece10": 0.290222452163805, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.4055555555555555, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 1, "fix": 1, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 1, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 3, "fix": 0, "refactor": 1, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.6357743613401263, "nll": 0.867789845499556, "ece10": 0.2692245519075021, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.43333333333333335, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 7, "accuracy": 0.4375, "wilson95": [ 0.23098652405492354, 0.6682144360118811 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4375, "brier": 0.6051981249379272, "nll": 0.8200760679057983, "ece10": 0.24903486993242047, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.41700404858299595, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 6, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 2, "accuracy": 0.1111111111111111, "wilson95": [ 0.03101952264561239, 0.32799765130178804 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 2, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.1111111111111111, "brier": 0.8619969015631695, "nll": 1.6987276944768035, "ece10": 0.28444691217218765, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.07777777777777777, "confusion_matrix": { "harassment": { "harassment": 0, "threat": 0, "self_harm": 4, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 0, "self_harm": 2, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 3, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "none": { "harassment": 1, "threat": 0, "self_harm": 4, "spam": 0, "none": 1, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 5, "accuracy": 0.22727272727272727, "wilson95": [ 0.10123039985248516, 0.4343995317522171 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.22727272727272727, "brier": 0.7130709780580147, "nll": 1.4130946475189363, "ece10": 0.2560556186500385, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.13, "confusion_matrix": { "news_report": { "news_report": 0, "opinion": 0, "press_release": 5, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 1, "opinion": 0, "press_release": 2, "analysis": 2, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 1, "opinion": 0, "press_release": 2, "analysis": 1, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 0, "opinion": 0, "press_release": 3, "analysis": 1, "feature": 0, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.40568437066062757, "nll": 0.797336519123385, "ece10": 0.15593627068406998, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5576190476190476, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 1, "termination": 2, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 1, "confidentiality": 1, "termination": 0, "indemnification": 2, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 1, "confidentiality": 0, "termination": 1, "indemnification": 1, "force_majeure": 0, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 9, "accuracy": 0.47368421052631576, "wilson95": [ 0.2732980516117956, 0.6829219075488682 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47368421052631576, "brier": 0.5859004685690745, "nll": 0.975779543027317, "ece10": 0.23339938678654115, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.4133333333333333, "confusion_matrix": { "address_issue": { "address_issue": 1, "customs_hold": 3, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 3, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 2, "weather_delay": 0, "failed_attempt": 1, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 2, "weather_delay": 0, "failed_attempt": 0, "damaged": 2, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.5050482866058423, "nll": 0.6981907667281971, "ece10": 0.06926147893608649, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.47058823529411764, "confusion_matrix": { "false": { "false": 4, "true": 5, "__failed__": 0 }, "true": { "false": 4, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 10, "accuracy": 0.6666666666666666, "wilson95": [ 0.41713547738519774, 0.8482367556028526 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.4209982260628821, "nll": 0.8245101367507235, "ece10": 0.26344583765156004, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.6805194805194805, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 2, "purchase_order": 0, "receipt": 1, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 1, "purchase_order": 0, "receipt": 0, "contract": 3, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 2, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 1, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 12, "accuracy": 0.8, "wilson95": [ 0.5481455128483065, 0.9295245065301844 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.4411435680837449, "nll": 0.8692832151653952, "ece10": 0.2813055016204329, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7814285714285714, "confusion_matrix": { "meeting_request": { "meeting_request": 1, "question": 0, "complaint": 0, "thanks": 1, "fyi": 1, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 2, "thanks": 0, "fyi": 1, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 9, "accuracy": 0.5, "wilson95": [ 0.29031021505169974, 0.7096897849483003 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6411313557731341, "nll": 1.1961625463399574, "ece10": 0.08373773127150184, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5133333333333333, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 2, "travel": 3, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 1, "travel": 1, "office_supplies": 2, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 3, "travel": 0, "office_supplies": 0, "software": 1, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 1, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 8, "accuracy": 0.4444444444444444, "wilson95": [ 0.24559518127005633, 0.6628358435795723 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4444444444444444, "brier": 0.709985347729989, "nll": 0.9407226526601941, "ece10": 0.36689829406838215, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.41558441558441556, "confusion_matrix": { "false": { "false": 6, "true": 3, "__failed__": 0 }, "true": { "false": 7, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 4, "accuracy": 0.25, "wilson95": [ 0.10182067491213048, 0.49498316535508774 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.25, "brier": 0.7526455644623039, "nll": 1.3650084250025156, "ece10": 0.174729375571645, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.18333333333333335, "confusion_matrix": { "0": { "0": 1, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 10, "accuracy": 0.5555555555555556, "wilson95": [ 0.33716415642042774, 0.7544048187299437 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5555555555555556, "brier": 0.5615022601848936, "nll": 0.9362967243815346, "ece10": 0.11241518546226789, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.48611111111111116, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 2, "1": 0, "2": 5, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 8, "accuracy": 0.4, "wilson95": [ 0.21880653237281705, 0.6134184992377469 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.6089394780012449, "nll": 1.1672217721416813, "ece10": 0.13081892406386933, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.39, "confusion_matrix": { "cheating": { "cheating": 1, "harassment": 1, "boosting": 0, "smurfing": 0, "other": 2, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 1, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 1, "other": 0, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 1, "smurfing": 0, "other": 2, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 2, "boosting": 0, "smurfing": 2, "other": 1, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 5, "accuracy": 0.23809523809523808, "wilson95": [ 0.10628013022636468, 0.4509117368295075 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.23809523809523808, "brier": 0.818313034347723, "nll": 1.6695864295292953, "ece10": 0.10657672725192725, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.13333333333333333, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 4, "punctuation": 0, "none": 0, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 3, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 4, "punctuation": 0, "none": 0, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 5, "punctuation": 0, "none": 0, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 4, "punctuation": 0, "none": 1, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.4203133206188345, "nll": 0.6110594315858546, "ece10": 0.10841594100857749, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6862745098039216, "confusion_matrix": { "false": { "false": 5, "true": 2, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 19, "accuracy": 0.8260869565217391, "wilson95": [ 0.6286235235977385, 0.9302134643693744 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8260869565217391, "brier": 0.3553748994344899, "nll": 0.7301322173035377, "ece10": 0.3297397241606378, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 0.814957264957265, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 3, "hvac": 1, "handyman": 1, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 2, "appliance_repair": 2, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 4, "accuracy": 0.2857142857142857, "wilson95": [ 0.11721378644576919, 0.5464908432989318 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2857142857142857, "brier": 0.8693367561126365, "nll": 1.8252420390038913, "ece10": 0.14696356719853673, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.19487179487179487, "confusion_matrix": { "0": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 0, "2": 0, "3": 5, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 3, "accuracy": 0.1875, "wilson95": [ 0.06591599071428142, 0.4300888096197414 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 3, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.1875, "brier": 0.8153463951341998, "nll": 1.4758010580956127, "ece10": 0.2867440644757405, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.08333333333333333, "confusion_matrix": { "0": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.677483130503831, "nll": 1.2443027535931845, "ece10": 0.13643282697284603, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.4303571428571429, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 1, "1": 1, "2": 1, "3": 1, "__failed__": 0 }, "3": { "0": 1, "1": 0, "2": 0, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.511202403784467, "nll": 0.7060630145832546, "ece10": 0.12670441282098252, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.34615384615384615, "confusion_matrix": { "false": { "false": 0, "true": 8, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.16973978179563692, "nll": 0.4470680709903536, "ece10": 0.2247119657284945, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8877777777777778, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 1, "business": 3, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "sports": 2, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.5330995580652296, "nll": 0.9753694450264615, "ece10": 0.23417558456937582, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.48, "confusion_matrix": { "payments": { "payments": 1, "web": 3, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 3, "infra": 1, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 2, "infra": 0, "data": 1, "model": 0, "__failed__": 0 }, "model": { "payments": 1, "web": 0, "infra": 0, "data": 0, "model": 1, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.42047897571356213, "nll": 0.6075218543510543, "ece10": 0.11335034591835431, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6886446886446886, "confusion_matrix": { "false": { "false": 4, "true": 4, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.45880281259426137, "nll": 0.6497466151133883, "ece10": 0.22380123495894716, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.45893719806763283, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 7, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 5, "accuracy": 0.3125, "wilson95": [ 0.14164643854782039, 0.5559564416525933 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3125, "brier": 0.6720824482353669, "nll": 1.0913115374467242, "ece10": 0.1712442139960349, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.15873015873015872, "confusion_matrix": { "0": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 6, "2": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 8, "accuracy": 0.4444444444444444, "wilson95": [ 0.24559518127005633, 0.6628358435795723 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4444444444444444, "brier": 0.6814805655241785, "nll": 1.6088869975320805, "ece10": 0.2869932523234223, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.40079365079365076, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 1, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 1, "indian": 2, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 1, "indian": 0, "thai": 1, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 2, "french": 0, "mexican": 1, "indian": 1, "thai": 0, "middle_eastern": 0, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 5, "accuracy": 0.29411764705882354, "wilson95": [ 0.1327998961029523, 0.5313311006653985 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.29411764705882354, "brier": 0.8116783053673412, "nll": 1.0779590187139345, "ece10": 0.41431508109256093, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.29166666666666663, "confusion_matrix": { "false": { "false": 2, "true": 6, "__failed__": 0 }, "true": { "false": 6, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 9, "accuracy": 0.5, "wilson95": [ 0.29031021505169974, 0.7096897849483003 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6425977686062353, "nll": 1.1600549538581957, "ece10": 0.3251419881050118, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.49963369963369964, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 2, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 2, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 4, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 3, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 8, "accuracy": 0.4444444444444444, "wilson95": [ 0.24559518127005633, 0.6628358435795723 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4444444444444444, "brier": 0.6969568597008601, "nll": 1.3824608202896742, "ece10": 0.17390157871742654, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.39428571428571424, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 1, "3": 0, "4": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 2, "4": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 5, "accuracy": 0.3125, "wilson95": [ 0.14164643854782039, 0.5559564416525933 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3125, "brier": 0.6828399973318178, "nll": 0.900360450573082, "ece10": 0.3239765074526469, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.2874493927125506, "confusion_matrix": { "false": { "false": 1, "true": 9, "__failed__": 0 }, "true": { "false": 2, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 7, "accuracy": 0.4666666666666667, "wilson95": [ 0.24809535365737387, 0.698830199745016 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.4909883262870068, "nll": 0.6841069028983987, "ece10": 0.056268875284015224, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.4642857142857143, "confusion_matrix": { "false": { "false": 3, "true": 5, "__failed__": 0 }, "true": { "false": 3, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.44182612124952597, "nll": 0.8382911630970038, "ece10": 0.15571695630924426, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7123809523809523, "confusion_matrix": { "billing": { "billing": 2, "tech": 2, "sales": 0, "account": 1, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 1, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.4526691301923162, "nll": 0.6448008825362708, "ece10": 0.023656632408756935, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5641025641025641, "confusion_matrix": { "false": { "false": 3, "true": 5, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 5, "accuracy": 0.2631578947368421, "wilson95": [ 0.11806412346782508, 0.487915508978151 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2631578947368421, "brier": 0.7980009385791668, "nll": 1.4731041349494118, "ece10": 0.09218898526220637, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.2048611111111111, "confusion_matrix": { "0": { "0": 0, "1": 6, "2": 0, "3": 1, "__failed__": 0 }, "1": { "0": 2, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 1, "1": 0, "2": 0, "3": 1, "__failed__": 0 }, "3": { "0": 1, "1": 2, "2": 0, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.42645189711528325, "nll": 0.6147154270909381, "ece10": 0.17992903203298882, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.39285714285714285, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.2206732240404228, "nll": 0.35581496682650066, "ece10": 0.27369973930299396, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9442724458204335, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 9, "accuracy": 0.45, "wilson95": [ 0.2581978582556243, 0.6579146575496577 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.6808523100777298, "nll": 1.2646076310479983, "ece10": 0.1338431676779148, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.41050420168067225, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 3, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 1, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 16, "accuracy": 0.8, "wilson95": [ 0.5839825677481064, 0.919342337420202 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.33201017756480666, "nll": 0.7160606033511158, "ece10": 0.2907988746972974, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7982905982905982, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 1, "information_request": 1, "reminder": 2, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 2, "reminder": 0, "music": 0, "small_talk": 2, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 13, "accuracy": 0.65, "wilson95": [ 0.43285427668523624, 0.818808175898918 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.5401794498670272, "nll": 0.7464434042302364, "ece10": 0.23458612285131417, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.6491228070175439, "confusion_matrix": { "false": { "false": 6, "true": 3, "__failed__": 0 }, "true": { "false": 4, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 7, "accuracy": 0.3181818181818182, "wilson95": [ 0.1636059318135624, 0.5268140225895723 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3181818181818182, "brier": 0.6429598377259012, "nll": 1.1799359402052088, "ece10": 0.3165824499039806, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.19871794871794873, "confusion_matrix": { "0": { "0": 0, "1": 1, "2": 0, "3": 4, "__failed__": 0 }, "1": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 393, "accuracy": 0.43666666666666665, "wilson95": [ 0.40460070187444824, 0.46927098337802514 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 504, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.46971109040074555, "cluster_bootstrap95": [ 0.3969729721539701, 0.4731615119170784 ], "independent_source_groups": 596, "brier": 0.5921354460488878, "nll": 0.9812380397591014, "ece10": 0.050691216921439, "calibration_n": 1073, "confident_wrong": 0 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 242, "accuracy": 0.42160278745644597, "wilson95": [ 0.38185736835724604, 0.46239056742832507 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 532, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5683760683760684, "brier": 0.5792343261914011, "nll": 1.1788021016372705, "ece10": 0.07781117711122285, "calibration_n": 936, "confident_wrong": 19 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 116, "accuracy": 0.1657142857142857, "wilson95": [ 0.1400089139479363, 0.19506861788125296 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 302, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2775735294117647, "cluster_bootstrap95": [ 0.13772820890184292, 0.19405583182179467 ], "independent_source_groups": 650, "brier": 0.7878505929206676, "nll": 1.5093460507592547, "ece10": 0.1785466319231413, "calibration_n": 1088, "confident_wrong": 21 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 7, "accuracy": 0.0175, "wilson95": [ 0.008502291107038103, 0.03567707292555804 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 966, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.483, "brier": 0.6246889298083192, "nll": 1.09258239590726, "ece10": 0.0727422710527654, "calibration_n": 2000, "confident_wrong": 2 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 22, "accuracy": 0.7857142857142857, "wilson95": [ 0.6046141434627812, 0.8978753966710211 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.29736742346641837, "nll": 0.5735206668105721, "ece10": 0.16790268870416084, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.7980769230769231, "confusion_matrix": { "design": { "design": 6, "inspection": 1, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 2, "inspection": 5, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 2, "inspection": 0, "purchase": 0, "work": 5, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 21, "accuracy": 0.75, "wilson95": [ 0.5664432968100104, 0.8732350508070668 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3467345763673542, "nll": 0.6477508398335815, "ece10": 0.15579751941271716, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.7476190476190475, "confusion_matrix": { "work": { "work": 6, "purchase": 0, "inspection": 0, "design": 1, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 6, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 1, "inspection": 4, "design": 1, "__failed__": 0 }, "design": { "work": 0, "purchase": 1, "inspection": 1, "design": 5, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 21, "accuracy": 0.75, "wilson95": [ 0.5664432968100104, 0.8732350508070668 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3258416809285289, "nll": 0.5910480563277727, "ece10": 0.14582865517036545, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.7564102564102564, "confusion_matrix": { "design": { "design": 6, "inspection": 1, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 2, "inspection": 4, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 2, "inspection": 0, "purchase": 0, "work": 5, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 18, "accuracy": 0.6428571428571429, "wilson95": [ 0.4583030651442781, 0.7929417049226233 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6428571428571429, "brier": 0.41944701964031156, "nll": 0.7422468132927642, "ece10": 0.18291782126636105, "calibration_n": 28, "confident_wrong": 1, "macro_f1": 0.609241452991453, "confusion_matrix": { "work": { "work": 6, "purchase": 0, "inspection": 0, "design": 1, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 6, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 4, "purchase": 0, "inspection": 1, "design": 2, "__failed__": 0 }, "design": { "work": 1, "purchase": 0, "inspection": 1, "design": 5, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 23, "accuracy": 0.575, "wilson95": [ 0.42195068541991065, 0.7149060744004962 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.575, "brier": 0.5590364097273162, "nll": 1.1908519614960065, "ece10": 0.1731939422018878, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.4560846560846561, "confusion_matrix": { "review": { "review": 0, "it": 5, "documents": 3, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 7, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 7, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 27, "accuracy": 0.675, "wilson95": [ 0.5201774618987768, 0.799154977682173 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.675, "brier": 0.5191788225081282, "nll": 1.107004020177723, "ece10": 0.12881618306235423, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.6183940242763774, "confusion_matrix": { "maintenance": { "maintenance": 3, "procurement": 0, "documents": 1, "it": 4, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 7, "review": 0, "__failed__": 0 }, "review": { "maintenance": 3, "procurement": 1, "documents": 2, "it": 1, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 24, "accuracy": 0.6, "wilson95": [ 0.44595893660346186, 0.7365167431570808 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 24, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5287854467294851, "nll": 1.1148670289987967, "ece10": 0.11847600686331296, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.5508300395256917, "confusion_matrix": { "review": { "review": 0, "it": 4, "documents": 4, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 6, "documents": 2, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 1, "documents": 7, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 1, "procurement": 7, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 3, "documents": 1, "procurement": 0, "maintenance": 4, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 27, "accuracy": 0.675, "wilson95": [ 0.5201774618987768, 0.799154977682173 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.675, "brier": 0.5153545895975047, "nll": 1.0666464469816646, "ece10": 0.13366638517686, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.6035562050907319, "confusion_matrix": { "maintenance": { "maintenance": 5, "procurement": 0, "documents": 1, "it": 2, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 6, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 1, "documents": 4, "it": 2, "review": 0, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 171, "accuracy": 0.4275, "wilson95": [ 0.3799345878043344, 0.47644469487413854 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 171, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4275, "brier": 0.7112235889736398, "nll": 1.475368834821988, "ece10": 0.09001294466905989, "calibration_n": 400, "confident_wrong": 3, "macro_f1": 0.3572016967776807, "confusion_matrix": { "sadness": { "sadness": 20, "joy": 6, "love": 20, "anger": 1, "fear": 71, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 0, "joy": 74, "love": 30, "anger": 0, "fear": 14, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 1, "joy": 6, "love": 23, "anger": 0, "fear": 7, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 5, "joy": 5, "love": 8, "anger": 11, "fear": 35, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 1, "joy": 2, "love": 3, "anger": 3, "fear": 42, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 5, "love": 1, "anger": 0, "fear": 1, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 210, "accuracy": 0.7, "wilson95": [ 0.6458824752950584, 0.749060336215123 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 210, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.47535929141004746, "nll": 1.1997552818253114, "ece10": 0.21393665300977574, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 315, "accuracy": 0.7875, "wilson95": [ 0.7447739406814162, 0.8247564900763633 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 315, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7875, "brier": 0.4363070353113343, "nll": 0.6304991425065585, "ece10": 0.2276907199379691, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.6299480838929461, "confusion_matrix": { "false": { "false": 288, "true": 21, "__failed__": 0 }, "true": { "false": 64, "true": 27, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 295, "accuracy": 0.7375, "wilson95": [ 0.6922684601360929, 0.7782132000551163 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 295, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7375, "brier": 0.3799784666757632, "nll": 0.7001322294729094, "ece10": 0.06974698412766211, "calibration_n": 400, "confident_wrong": 13, "macro_f1": 0.7152066205508714, "confusion_matrix": { "world": { "world": 86, "sports": 6, "business": 10, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 10, "sports": 112, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 21, "sports": 2, "business": 44, "sci_tech": 5, "__failed__": 0 }, "sci_tech": { "world": 20, "sports": 1, "business": 28, "sci_tech": 53, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 256, "accuracy": 0.64, "wilson95": [ 0.591834465196668, 0.685502092389729 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 256, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.64, "brier": 0.46793919994147765, "nll": 0.6627592860901997, "ece10": 0.056339482354545424, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.5511082016272327, "confusion_matrix": { "true": { "true": 39, "false": 114, "__failed__": 0 }, "false": { "true": 30, "false": 217, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 172, "accuracy": 0.5733333333333334, "wilson95": [ 0.5167859400590311, 0.6280264241613689 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 172, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5733333333333334, "brier": 0.5977855746524418, "nll": 1.5528768265776591, "ece10": 0.19168524194328654, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 211, "accuracy": 0.5275, "wilson95": [ 0.4785460270872868, 0.5759307967243268 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 211, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5275, "brier": 0.49731177308654856, "nll": 0.6909247363777392, "ece10": 0.06351600770922006, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.504037892554664, "confusion_matrix": { "false": { "false": 149, "true": 39, "__failed__": 0 }, "true": { "false": 150, "true": 62, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 229, "accuracy": 0.5725, "wilson95": [ 0.5235553051258613, 0.6200654121956656 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 229, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5725, "brier": 0.4854768753395288, "nll": 0.6788290273746374, "ece10": 0.022442250996879777, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.5211391846764539, "confusion_matrix": { "false": { "false": 180, "true": 20, "__failed__": 0 }, "true": { "false": 151, "true": 49, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 68, "accuracy": 0.68, "wilson95": [ 0.5833738138954762, 0.7633085267500092 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 68, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.68, "brier": 0.41853252128524693, "nll": 0.6091428243292263, "ece10": 0.07613143004638831, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.677938808373591, "confusion_matrix": { "yes": { "yes": 38, "no": 16, "__failed__": 0 }, "no": { "yes": 16, "no": 30, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 54, "accuracy": 0.54, "wilson95": [ 0.44264860323368227, 0.6343919169097589 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 54, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.54, "brier": 0.6114429587864678, "nll": 1.4461233603266674, "ece10": 0.14569581300559561, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5462146921705745, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 5, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 2, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 9, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 2, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 1, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 5, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 2, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 5, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 4, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 2, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 3, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 5, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 1, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 1, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 3, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 50, "accuracy": 0.5, "wilson95": [ 0.4038315303659956, 0.5961684696340044 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 143, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.715, "brier": 0.38154628516167827, "nll": 0.612124758766078, "ece10": 0.047704517120021266, "calibration_n": 200, "confident_wrong": 1 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 39, "accuracy": 0.39, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 124, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.62, "brier": 0.49498822166899076, "nll": 1.0031099507453909, "ece10": 0.05916017012730265, "calibration_n": 200, "confident_wrong": 2 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 72, "accuracy": 0.72, "wilson95": [ 0.6251197129007884, 0.7986031478881379 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 72, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.72, "brier": 0.4099014136657054, "nll": 1.0080896803813693, "ece10": 0.1615605247404755, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6975359801017696, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 3, "receipt": 12, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 3, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 9, "clinical_note": 0, "syllabus": 0, "news_article": 2, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 2, "__failed__": 0 }, "clinical_note": { "invoice": 1, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 5, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 9, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 5, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 12, "accuracy": 0.12, "wilson95": [ 0.06999406437019488, 0.19812099426711421 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 227, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5675, "brier": 0.5678531605762012, "nll": 1.04482518408183, "ece10": 0.055385685542109896, "calibration_n": 400, "confident_wrong": 0 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 56, "accuracy": 0.56, "wilson95": [ 0.4622810465167699, 0.6532797336983921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 56, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.56, "brier": 0.5607503150165871, "nll": 1.2082492917901835, "ece10": 0.08735908512764985, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.568720167099793, "confusion_matrix": { "politics": { "politics": 2, "business": 0, "technology": 2, "science": 2, "health": 4, "sports": 0, "entertainment": 0, "world": 3, "culture": 1, "weather": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 8, "technology": 0, "science": 1, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 2, "technology": 7, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 1, "business": 1, "technology": 0, "science": 5, "health": 1, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 9, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 1, "health": 1, "sports": 5, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 0, "business": 0, "technology": 1, "science": 0, "health": 1, "sports": 0, "entertainment": 6, "world": 0, "culture": 1, "weather": 1, "__failed__": 0 }, "world": { "politics": 2, "business": 1, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 4, "culture": 1, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 0, "world": 3, "culture": 6, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 1, "sports": 0, "entertainment": 0, "world": 2, "culture": 0, "weather": 4, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 40, "accuracy": 0.4, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.7733281821128994, "nll": 1.8420955832641652, "ece10": 0.07348808835453766, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.3842977569741741, "confusion_matrix": { "biology": { "biology": 8, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 3, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 2, "mathematics": 2, "earth_science": 0, "engineering": 0, "economics": 1, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 1, "medicine": 0, "chemistry": 3, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 3, "medicine": 0, "chemistry": 0, "physics": 9, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 2, "medicine": 1, "chemistry": 1, "physics": 3, "computer_science": 1, "mathematics": 0, "earth_science": 3, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 4, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 2, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 2, "mathematics": 0, "earth_science": 7, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 0, "chemistry": 1, "physics": 1, "computer_science": 2, "mathematics": 1, "earth_science": 0, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 1, "earth_science": 1, "engineering": 0, "economics": 4, "other": 0, "__failed__": 0 }, "other": { "biology": 1, "medicine": 0, "chemistry": 0, "physics": 4, "computer_science": 2, "mathematics": 1, "earth_science": 2, "engineering": 1, "economics": 1, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 14, "accuracy": 0.14, "wilson95": [ 0.08526343659939134, 0.22137188210963837 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.513997561539083, "nll": 1.0024354393753014, "ece10": 0.10404331041326939, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 3, "accuracy": 0.03, "wilson95": [ 0.010254524024038925, 0.0845193642905276 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.3655912571284311, "nll": 0.6215385358228908, "ece10": 0.07497173788443208, "calibration_n": 100, "confident_wrong": 2 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 78, "accuracy": 0.78, "wilson95": [ 0.6892964648501421, 0.8499871761539459 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 78, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.78, "brier": 0.2999110199642409, "nll": 0.5227852261448983, "ece10": 0.09377670450348123, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.7499054187765554, "confusion_matrix": { "negative": { "negative": 36, "neutral": 0, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 11, "neutral": 13, "positive": 9, "__failed__": 0 }, "positive": { "negative": 0, "neutral": 2, "positive": 29, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 1, "accuracy": 0.01, "wilson95": [ 0.0017674320641406505, 0.054486196178705315 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 78, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.39, "brier": 0.2786944560125645, "nll": 0.4360030440280781, "ece10": 0.039751269348775674, "calibration_n": 100, "confident_wrong": 2 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 23, "accuracy": 0.23, "wilson95": [ 0.15843265880303448, 0.3215438302287378 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 193, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6433333333333333, "brier": 0.468025980966598, "nll": 0.8978350588623806, "ece10": 0.07448752226214676, "calibration_n": 300, "confident_wrong": 4 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 63, "accuracy": 0.63, "wilson95": [ 0.5322052958094291, 0.7181763946567546 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 63, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.63, "brier": 0.5200709209672174, "nll": 1.419574000018503, "ece10": 0.1255796180539006, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.590454223647501, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 3, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 2, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 1, "password_reset": 5, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 5, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 2, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 3, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 1, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 1, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 2, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 2, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 4, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 6, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 2, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 46, "accuracy": 0.46, "wilson95": [ 0.36560808309024106, 0.5573513967663177 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 46, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.46, "brier": 0.714499918755586, "nll": 1.688723241020143, "ece10": 0.12917053385966573, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.446219384480254, "confusion_matrix": { "billing": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 3, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 6, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 1, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 4, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 2, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 1, "account_access": 2, "shipping": 2, "returns": 0, "technical_outage": 1, "onboarding": 0, "integrations": 3, "security": 1, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 4, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 6, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 2, "compliance": 1, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "compliance": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 2, "pricing": 0, "data_privacy": 1, "other": 1, "__failed__": 0 }, "pricing": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 5, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 6, "other": 0, "__failed__": 0 }, "other": { "billing": 3, "account_access": 0, "shipping": 2, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 1, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 12, "accuracy": 0.12, "wilson95": [ 0.06999406437019488, 0.19812099426711421 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 139, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4633333333333333, "brier": 0.6202894769985289, "nll": 1.2932963187617834, "ece10": 0.05149248927989619, "calibration_n": 300, "confident_wrong": 0 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 52, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.6178231475203806, "nll": 1.3732366996843677, "ece10": 0.12428614580746512, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.4991431444372621, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 2, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 1, "change_flight": 4, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 3, "visa_docs": 0, "loyalty": 1, "disruption": 0, "other": 2, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 2, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 6, "baggage": 0, "refund": 0, "hotel": 0, "car": 2, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 11, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 3, "hotel": 0, "car": 2, "visa_docs": 0, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 9, "car": 0, "visa_docs": 1, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 4, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 6, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 3, "disruption": 3, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 2, "visa_docs": 2, "loyalty": 3, "disruption": 2, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 79, "accuracy": 0.79, "wilson95": [ 0.7002003116591013, 0.858343459380847 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 79, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.31696714820996436, "nll": 0.5797343167545921, "ece10": 0.05702970310588292, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.7856507433169667, "confusion_matrix": { "business": { "business": 17, "science and technology": 0, "sports": 1, "world": 7, "__failed__": 0 }, "science and technology": { "business": 5, "science and technology": 14, "sports": 1, "world": 5, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 23, "world": 2, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 34, "accuracy": 0.34, "wilson95": [ 0.25461520797348164, 0.43722271145275377 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.34, "brier": 0.7780967622453068, "nll": 2.420017536183626, "ece10": 0.12820267614355974, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.34807692307692306, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 1, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 1, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 1, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 1, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 1, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 50, "accuracy": 0.5, "wilson95": [ 0.4038315303659956, 0.5961684696340044 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 50, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.7075040505062993, "nll": 1.464633021542354, "ece10": 0.14577700675532212, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.45981042206972883, "confusion_matrix": { "anger": { "anger": 2, "fear": 11, "joy": 2, "love": 1, "sadness": 1, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 0, "fear": 15, "joy": 1, "love": 0, "sadness": 0, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 11, "love": 5, "sadness": 1, "surprise": 0, "__failed__": 0 }, "love": { "anger": 0, "fear": 1, "joy": 3, "love": 12, "sadness": 1, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 5, "joy": 3, "love": 3, "sadness": 5, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 0, "fear": 1, "joy": 5, "love": 3, "sadness": 2, "surprise": 5, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1625, "accuracy": 0.5512211668928086, "wilson95": [ 0.5332119780749904, 0.5690970396053336 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1625, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5512211668928086, "brier": 0.6398682588922375, "nll": 1.6130873986295444, "ece10": 0.18921730609620613, "calibration_n": 2948, "confident_wrong": 0, "macro_f1": 0.5656287231524728, "confusion_matrix": { "alarm": { "alarm": 85, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 6, "email": 0, "general": 0, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 3, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 7, "audio": 34, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 14, "iot": 1, "lists": 0, "music": 3, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 7, "audio": 0, "calendar": 240, "cooking": 9, "datetime": 43, "email": 3, "general": 53, "iot": 2, "lists": 3, "music": 5, "news": 3, "play": 1, "qa": 1, "recommendation": 17, "social": 2, "takeaway": 10, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 66, "datetime": 0, "email": 0, "general": 0, "iot": 1, "lists": 3, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 13, "cooking": 0, "datetime": 81, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 3, "email": 229, "general": 23, "iot": 1, "lists": 3, "music": 0, "news": 1, "play": 0, "qa": 0, "recommendation": 1, "social": 3, "takeaway": 0, "transport": 1, "weather": 3, "__failed__": 0 }, "general": { "alarm": 0, "audio": 5, "calendar": 2, "cooking": 9, "datetime": 20, "email": 1, "general": 96, "iot": 10, "lists": 3, "music": 4, "news": 2, "play": 14, "qa": 1, "recommendation": 10, "social": 0, "takeaway": 4, "transport": 1, "weather": 5, "__failed__": 0 }, "iot": { "alarm": 23, "audio": 1, "calendar": 0, "cooking": 42, "datetime": 2, "email": 0, "general": 81, "iot": 52, "lists": 1, "music": 4, "news": 0, "play": 1, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 3, "transport": 0, "weather": 2, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 3, "datetime": 0, "email": 1, "general": 11, "iot": 2, "lists": 117, "music": 4, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 74, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 1, "audio": 1, "calendar": 0, "cooking": 1, "datetime": 4, "email": 0, "general": 21, "iot": 0, "lists": 1, "music": 0, "news": 93, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "play": { "alarm": 1, "audio": 33, "calendar": 0, "cooking": 1, "datetime": 4, "email": 0, "general": 18, "iot": 3, "lists": 3, "music": 183, "news": 1, "play": 136, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 4, "datetime": 33, "email": 2, "general": 172, "iot": 11, "lists": 5, "music": 2, "news": 1, "play": 10, "qa": 3, "recommendation": 29, "social": 1, "takeaway": 0, "transport": 3, "weather": 9, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 16, "email": 0, "general": 20, "iot": 1, "lists": 7, "music": 4, "news": 2, "play": 8, "qa": 0, "recommendation": 34, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 1, "datetime": 0, "email": 4, "general": 30, "iot": 12, "lists": 0, "music": 0, "news": 3, "play": 0, "qa": 0, "recommendation": 0, "social": 48, "takeaway": 5, "transport": 0, "weather": 1, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 8, "datetime": 3, "email": 0, "general": 2, "iot": 1, "lists": 1, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 41, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 13, "email": 0, "general": 16, "iot": 0, "lists": 3, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 9, "social": 0, "takeaway": 5, "transport": 76, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 8, "email": 0, "general": 14, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 3, "social": 0, "takeaway": 6, "transport": 0, "weather": 120, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1455, "accuracy": 0.4935549525101764, "wilson95": [ 0.47552753436167267, 0.5115991455328561 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1455, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4935549525101764, "brier": 0.6739740486406965, "nll": 1.7245618931136175, "ece10": 0.14595367389662195, "calibration_n": 2948, "confident_wrong": 0, "macro_f1": 0.5169003806223551, "confusion_matrix": { "alarm": { "alarm": 84, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 5, "email": 0, "general": 2, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 0, "audio": 34, "calendar": 0, "cooking": 0, "datetime": 5, "email": 0, "general": 17, "iot": 1, "lists": 0, "music": 3, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 4, "audio": 0, "calendar": 197, "cooking": 10, "datetime": 59, "email": 6, "general": 95, "iot": 0, "lists": 3, "music": 4, "news": 2, "play": 1, "qa": 1, "recommendation": 3, "social": 2, "takeaway": 11, "transport": 1, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 67, "datetime": 0, "email": 0, "general": 0, "iot": 1, "lists": 3, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 1, "audio": 0, "calendar": 14, "cooking": 0, "datetime": 79, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 11, "email": 177, "general": 52, "iot": 4, "lists": 13, "music": 0, "news": 3, "play": 0, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 1, "weather": 3, "__failed__": 0 }, "general": { "alarm": 0, "audio": 5, "calendar": 3, "cooking": 11, "datetime": 15, "email": 2, "general": 105, "iot": 11, "lists": 3, "music": 6, "news": 1, "play": 5, "qa": 0, "recommendation": 7, "social": 0, "takeaway": 7, "transport": 2, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 28, "audio": 0, "calendar": 0, "cooking": 41, "datetime": 5, "email": 0, "general": 98, "iot": 31, "lists": 0, "music": 3, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 3, "transport": 0, "weather": 4, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 11, "calendar": 3, "cooking": 2, "datetime": 1, "email": 1, "general": 12, "iot": 2, "lists": 107, "music": 2, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 2, "music": 71, "news": 0, "play": 1, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 7, "email": 0, "general": 31, "iot": 0, "lists": 3, "music": 0, "news": 77, "play": 0, "qa": 0, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "play": { "alarm": 0, "audio": 32, "calendar": 0, "cooking": 1, "datetime": 7, "email": 0, "general": 20, "iot": 4, "lists": 24, "music": 153, "news": 0, "play": 144, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 2, "calendar": 5, "cooking": 5, "datetime": 36, "email": 6, "general": 166, "iot": 7, "lists": 4, "music": 1, "news": 2, "play": 12, "qa": 3, "recommendation": 24, "social": 0, "takeaway": 0, "transport": 3, "weather": 11, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 2, "calendar": 3, "cooking": 4, "datetime": 8, "email": 0, "general": 36, "iot": 2, "lists": 2, "music": 4, "news": 0, "play": 9, "qa": 0, "recommendation": 22, "social": 0, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "social": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 2, "datetime": 0, "email": 10, "general": 29, "iot": 9, "lists": 2, "music": 0, "news": 1, "play": 0, "qa": 0, "recommendation": 2, "social": 43, "takeaway": 6, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 11, "datetime": 2, "email": 0, "general": 6, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 36, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 1, "datetime": 10, "email": 0, "general": 16, "iot": 0, "lists": 3, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 4, "social": 0, "takeaway": 4, "transport": 82, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 0, "datetime": 13, "email": 0, "general": 21, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 8, "social": 0, "takeaway": 7, "transport": 1, "weather": 96, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 780, "accuracy": 0.6649616368286445, "wilson95": [ 0.6374506650974257, 0.6913956693848846 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 780, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6649616368286445, "cluster_bootstrap95": [ 0.6240645807792237, 0.7015988103228671 ], "independent_source_groups": 47, "brier": 0.4725626843413911, "nll": 0.8080224749434048, "ece10": 0.05316036577124424, "calibration_n": 1173, "confident_wrong": 8, "macro_f1": 0.5686267252181841, "confusion_matrix": { "negative": { "negative": 36, "neutral": 121, "positive": 25, "__failed__": 0 }, "neutral": { "negative": 21, "neutral": 455, "positive": 114, "__failed__": 0 }, "positive": { "negative": 7, "neutral": 105, "positive": 289, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "gliner-decide": { "label": "GLiNER Decide 340M", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 140, "accuracy": 0.7, "wilson95": [ 0.6332093162728074, 0.7592525531762859 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 140, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.46181483637554555, "nll": 1.0231635732463957, "ece10": 0.10842636268729124, "calibration_n": 200, "confident_wrong": 10, "macro_f1": 0.6783001587784196, "confusion_matrix": { "world": { "world": 47, "sports": 0, "business": 2, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 3, "sports": 43, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 14, "sports": 0, "business": 37, "sci_tech": 0, "__failed__": 0 }, "sci_tech": { "world": 11, "sports": 2, "business": 26, "sci_tech": 13, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 106, "accuracy": 0.53, "wilson95": [ 0.4609168291500183, 0.5979524512673458 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 106, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.53, "brier": 0.6373159492610412, "nll": 1.3092263534426385, "ece10": 0.11883441639965502, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.4057052566533597, "confusion_matrix": { "sadness": { "sadness": 45, "joy": 3, "love": 3, "anger": 4, "fear": 2, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 13, "joy": 33, "love": 28, "anger": 3, "fear": 2, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 4, "joy": 1, "love": 7, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 11, "joy": 1, "love": 1, "anger": 15, "fear": 0, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 7, "joy": 2, "love": 2, "anger": 2, "fear": 6, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 1, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 143, "accuracy": 0.715, "wilson95": [ 0.6488465397039482, 0.7730499699538271 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 143, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.715, "brier": 0.39829003405981633, "nll": 0.5875659991433335, "ece10": 0.10210683048874719, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.6739037157813439, "confusion_matrix": { "false": { "false": 107, "true": 2, "__failed__": 0 }, "true": { "false": 55, "true": 36, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 127, "accuracy": 0.635, "wilson95": [ 0.566317026527406, 0.698594735350732 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 127, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.635, "brier": 0.43161843799253935, "nll": 0.6219993691424527, "ece10": 0.09804132526286866, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.48863437357710765, "confusion_matrix": { "true": { "true": 10, "false": 67, "__failed__": 0 }, "false": { "true": 6, "false": 117, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.45950165110345925, "nll": 0.8515756830663728, "ece10": 0.13498593924554103, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.7066666666666668, "confusion_matrix": { "maintenance": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 4, "accuracy": 0.3333333333333333, "wilson95": [ 0.13812009109121315, 0.6093779111272004 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3333333333333333, "brier": 0.8155431332297199, "nll": 1.4317290242040839, "ece10": 0.3277610479409616, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.31333333333333335, "confusion_matrix": { "maintenance": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 1, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 1, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 1, "documents": 2, "it": 0, "review": 0, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.28506554812081203, "nll": 0.516135098576049, "ece10": 0.2513718176225002, "calibration_n": 12, "confident_wrong": 1, "macro_f1": 0.8285714285714286, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 2, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 2, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.1575278809792758, "nll": 0.31752728942785874, "ece10": 0.2392662262799873, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9142857142857144, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 1, "purchase": 2, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 87, "accuracy": 0.87, "wilson95": [ 0.7901964855992357, 0.9224283257275948 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 87, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.24659310599226766, "nll": 0.4878201931154671, "ece10": 0.08354357530718462, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.8597705802968961, "confusion_matrix": { "label_000": { "label_000": 25, "label_001": 0, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 9, "label_001": 13, "label_002": 1, "label_003": 2, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 24, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 56, "accuracy": 0.56, "wilson95": [ 0.4622810465167699, 0.6532797336983921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 56, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.56, "brier": 0.6174855949858955, "nll": 1.3122682205687546, "ece10": 0.13316071220154324, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5575194821518351, "confusion_matrix": { "label_000": { "label_000": 12, "label_001": 1, "label_002": 0, "label_003": 1, "label_004": 1, "label_005": 2, "__failed__": 0 }, "label_001": { "label_000": 2, "label_001": 11, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 2, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 7, "label_003": 7, "label_004": 1, "label_005": 1, "__failed__": 0 }, "label_003": { "label_000": 2, "label_001": 0, "label_002": 3, "label_003": 10, "label_004": 2, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 3, "label_001": 3, "label_002": 0, "label_003": 1, "label_004": 9, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 2, "label_001": 0, "label_002": 1, "label_003": 3, "label_004": 3, "label_005": 7, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 75, "accuracy": 0.75, "wilson95": [ 0.656955364519384, 0.8245478863771232 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 75, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3886990823056272, "nll": 1.1457760257668432, "ece10": 0.10539613216590737, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.7232142857142857, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 1, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 2, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 1, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 1, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 2, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 1, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 2, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 1, "label_026": 1, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 1, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 47, "accuracy": 0.9791666666666666, "wilson95": [ 0.8910078200474814, 0.9963128890224893 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 47, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9791666666666666, "brier": 0.07087629212403115, "nll": 0.18937972064991707, "ece10": 0.161764880937492, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 36, "accuracy": 0.32432432432432434, "wilson95": [ 0.24438273167344465, 0.41601865646207165 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 36, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.32432432432432434, "brier": 0.9517977178043768, "nll": 2.028460853200249, "ece10": 0.3735449863532271, "calibration_n": 111, "confident_wrong": 20 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 48, "accuracy": 0.6666666666666666, "wilson95": [ 0.5517960435660519, 0.764653565568462 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "cluster_bootstrap95": [ 0.5416666666666666, 0.7777777777777778 ], "independent_source_groups": 36, "brier": 0.37757571581556826, "nll": 0.6104420077670749, "ece10": 0.09136990883188927, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 15, "accuracy": 0.7894736842105263, "wilson95": [ 0.5666572155618583, 0.9149232336708375 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7894736842105263, "brier": 0.24704767601744676, "nll": 0.41074233479455835, "ece10": 0.2425031437784048, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.7840909090909092, "confusion_matrix": { "false": { "false": 9, "true": 2, "__failed__": 0 }, "true": { "false": 2, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.44834850287583955, "nll": 0.6362546667122305, "ece10": 0.032592129256485705, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5636363636363637, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 12, "accuracy": 0.5454545454545454, "wilson95": [ 0.3465979532073494, 0.7307970581918668 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5454545454545454, "brier": 0.5196192564149041, "nll": 0.7136931209778394, "ece10": 0.1502178863328997, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.47619047619047616, "confusion_matrix": { "false": { "false": 10, "true": 0, "__failed__": 0 }, "true": { "false": 10, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.14417424781850668, "nll": 0.3459151505838366, "ece10": 0.1512708429886056, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9415584415584415, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 3, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.29571779844669394, "nll": 0.6072549954894713, "ece10": 0.14616175539351942, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8428571428571429, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "4": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06530558054004268, "nll": 0.21576011776845405, "ece10": 0.18241257655996826, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.39014565841668425, "nll": 0.8245404785722602, "ece10": 0.2913736659517451, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7222222222222222, "confusion_matrix": { "bug": { "bug": 2, "performance": 0, "security": 0, "style": 1, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 1, "performance": 1, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 1, "refactor": 0, "nit": 2, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 1, "refactor": 0, "nit": 3, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.250275295315345, "nll": 0.503432701912305, "ece10": 0.208805668729088, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8277777777777778, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 1, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.42703586738765686, "nll": 0.6172101227904523, "ece10": 0.145526945385603, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6136363636363636, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.3252918615089289, "nll": 0.5058547593172562, "ece10": 0.17801622446959914, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8117647058823529, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.38115768686192053, "nll": 0.7388151714135278, "ece10": 0.25074908016201725, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7262626262626262, "confusion_matrix": { "harassment": { "harassment": 5, "threat": 0, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "threat": { "harassment": 1, "threat": 1, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 2, "spam": 1, "none": 3, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 18, "accuracy": 0.8181818181818182, "wilson95": [ 0.6148339256287345, 0.9269311541657794 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8181818181818182, "brier": 0.2959253884536712, "nll": 0.6011612757910746, "ece10": 0.26783513063543674, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8253968253968255, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 1, "opinion": 4, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 1, "opinion": 0, "press_release": 0, "analysis": 3, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 2, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 2, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.09198726319361633, "nll": 0.24498654580967277, "ece10": 0.19999895451537314, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9377777777777778, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 1, "termination": 2, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.058648181321358274, "nll": 0.14627662645199355, "ece10": 0.11391552666053117, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "address_issue": { "address_issue": 3, "customs_hold": 1, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.5007073327511501, "nll": 0.6934615619384398, "ece10": 0.18545345201463562, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.43333333333333335, "confusion_matrix": { "false": { "false": 1, "true": 8, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.128129199801261, "nll": 0.33031838739217295, "ece10": 0.2602549298631343, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.3172395203613287, "nll": 0.6255460725707294, "ece10": 0.32614744769695897, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8428571428571429, "confusion_matrix": { "meeting_request": { "meeting_request": 1, "question": 0, "complaint": 0, "thanks": 1, "fyi": 1, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.3309026552033895, "nll": 0.6570909464620295, "ece10": 0.22516354296130883, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7644444444444444, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 4, "office_supplies": 0, "software": 0, "client_entertainment": 1, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 1, "office_supplies": 1, "software": 2, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.3891021510478845, "nll": 0.5785402333227471, "ece10": 0.17301652816380286, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7662337662337663, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.49453302917935155, "nll": 0.8838688240014747, "ece10": 0.19543523150622588, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6464285714285714, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 5, "3": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.4162258502637462, "nll": 0.6930641113952019, "ece10": 0.10101450967669565, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6984126984126985, "confusion_matrix": { "0": { "0": 2, "1": 3, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 2, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 15, "accuracy": 0.75, "wilson95": [ 0.531299122381256, 0.8881382985923343 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3282774256586815, "nll": 0.6722859988159182, "ece10": 0.1703212789133534, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7657720057720058, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 4, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 2, "other": 1, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 0, "other": 2, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 6, "accuracy": 0.2857142857142857, "wilson95": [ 0.1381386295212213, 0.49956380716085597 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2857142857142857, "brier": 0.7795198313919192, "nll": 1.5636791062628241, "ece10": 0.036155084625072796, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.21999999999999997, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 1, "none": 3, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 0, "none": 2, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 2, "punctuation": 1, "none": 1, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 2, "none": 2, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 3, "none": 2, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.4493689221648406, "nll": 0.6404717981943513, "ece10": 0.107163319190944, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6135265700483092, "confusion_matrix": { "false": { "false": 2, "true": 5, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 20, "accuracy": 0.8695652173913043, "wilson95": [ 0.6787251772990404, 0.9546234090636876 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8695652173913043, "brier": 0.22024869542444223, "nll": 0.48627859238371124, "ece10": 0.21567677322404624, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 0.8632467532467534, "confusion_matrix": { "plumbing": { "plumbing": 4, "electrical": 0, "hvac": 0, "handyman": 1, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 3, "handyman": 0, "appliance_repair": 1, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 1, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 3, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 11, "accuracy": 0.7857142857142857, "wilson95": [ 0.524107694133997, 0.9242861328730683 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.36411979951923346, "nll": 0.6998222795194231, "ece10": 0.2701191904194418, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.6466666666666667, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.35301937936707956, "nll": 0.6582586030932267, "ece10": 0.1989783360212481, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7208791208791209, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.37558098520769423, "nll": 0.6854942481358691, "ece10": 0.1755916100319862, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7446428571428572, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.4920591050520476, "nll": 0.6851994350373074, "ece10": 0.04458466488074425, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5296442687747036, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 7, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.24503595522862412, "nll": 0.5140116374340501, "ece10": 0.11118979324503883, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.8806349206349205, "confusion_matrix": { "politics": { "politics": 3, "business": 0, "technology": 0, "sports": 0, "culture": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "sports": 2, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.2323084499557228, "nll": 0.4818768029224932, "ece10": 0.3609164060730568, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.2649871618288464, "nll": 0.4413996533176811, "ece10": 0.27044748580154454, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9403508771929825, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.4598115045766083, "nll": 0.6526126255515357, "ece10": 0.08775309115417892, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5636363636363637, "confusion_matrix": { "false": { "false": 2, "true": 6, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.4089811140154428, "nll": 0.7102517338094249, "ece10": 0.2601614552980617, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7452991452991453, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "__failed__": 0 }, "2": { "0": 2, "1": 1, "2": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.21063760840251539, "nll": 0.5259256730497267, "ece10": 0.20109338748784358, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8873015873015873, "confusion_matrix": { "italian": { "italian": 1, "french": 1, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 1, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 5, "accuracy": 0.29411764705882354, "wilson95": [ 0.1327998961029523, 0.5313311006653985 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.29411764705882354, "brier": 0.7884001954581712, "nll": 1.038004156118692, "ece10": 0.4428151272739591, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.27142857142857146, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 5, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.25542663211175215, "nll": 0.5360790385146273, "ece10": 0.27046696155964156, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.44492887642517853, "nll": 0.850873013890919, "ece10": 0.09618853938550172, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6409523809523809, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 1, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 6, "accuracy": 0.375, "wilson95": [ 0.18481232558863633, 0.6135895945449727 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.375, "brier": 0.5695573084026943, "nll": 0.7648176401465991, "ece10": 0.2716217443748334, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.2727272727272727, "confusion_matrix": { "false": { "false": 6, "true": 4, "__failed__": 0 }, "true": { "false": 6, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 9, "accuracy": 0.6, "wilson95": [ 0.357468301206246, 0.8017550385865841 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.4806389004501963, "nll": 0.6737070508428895, "ece10": 0.06686650399245157, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.5982142857142857, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 3, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.2671676384487439, "nll": 0.5717208941633367, "ece10": 0.33381046351931193, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8377777777777778, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 4, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 1, "account": 0, "other": 1, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.3939313191643595, "nll": 0.5855876256458805, "ece10": 0.3096359362640195, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8819444444444444, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 8, "accuracy": 0.42105263157894735, "wilson95": [ 0.23141891282454238, 0.6372409646574496 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42105263157894735, "brier": 0.6313375778904197, "nll": 1.1311206167077625, "ece10": 0.22805462584075303, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.23550724637681159, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 2, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 2, "1": 0, "2": 2, "3": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 6, "accuracy": 0.35294117647058826, "wilson95": [ 0.17309720397243744, 0.5869963651478132 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35294117647058826, "brier": 0.5725025295233883, "nll": 0.7680995685772923, "ece10": 0.24856141760737796, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.2608695652173913, "confusion_matrix": { "false": { "false": 0, "true": 11, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.2269113877893108, "nll": 0.3701664763810386, "ece10": 0.2175119737809138, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.775, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 14, "accuracy": 0.7, "wilson95": [ 0.4810271816464765, 0.8545227551323956 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.63084454744205, "nll": 1.1614254677527451, "ece10": 0.3522583740299448, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.6597222222222222, "confusion_matrix": { "0": { "0": 6, "1": 0, "2": 1, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 1, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 1, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 18, "accuracy": 0.9, "wilson95": [ 0.6989663547715128, 0.9721335187862319 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.2049777679381465, "nll": 0.4148529369769126, "ece10": 0.15239129728547618, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8984126984126984, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 4, "reminder": 0, "music": 0, "small_talk": 1, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 10, "accuracy": 0.5, "wilson95": [ 0.2992980081982123, 0.7007019918017877 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.4623004347440792, "nll": 0.6537364134369819, "ece10": 0.3207713340541281, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.40476190476190477, "confusion_matrix": { "false": { "false": 1, "true": 8, "__failed__": 0 }, "true": { "false": 2, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.491499025922347, "nll": 0.9366596828387469, "ece10": 0.4937626670029855, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9078282828282829, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 1, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 443, "accuracy": 0.4922222222222222, "wilson95": [ 0.45966263147181813, 0.5248479263662048 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 588, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5479962721342032, "cluster_bootstrap95": [ 0.4651800180892031, 0.5187178320024085 ], "independent_source_groups": 596, "brier": 0.5544771840956036, "nll": 0.9238761135381958, "ece10": 0.03662725413065144, "calibration_n": 1073, "confident_wrong": 0 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 237, "accuracy": 0.41289198606271776, "wilson95": [ 0.37332286848000046, 0.45361928239285637 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 528, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5641025641025641, "brier": 0.5876939017658933, "nll": 1.183133131070113, "ece10": 0.08925426786097801, "calibration_n": 936, "confident_wrong": 16 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 162, "accuracy": 0.23142857142857143, "wilson95": [ 0.20170252730236043, 0.26408625895356946 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 324, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2977941176470588, "cluster_bootstrap95": [ 0.20142857142857143, 0.26225017579657184 ], "independent_source_groups": 650, "brier": 0.8899139330739239, "nll": 2.428863642282676, "ece10": 0.2642556214660641, "calibration_n": 1088, "confident_wrong": 136 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 12, "accuracy": 0.03, "wilson95": [ 0.017243020965143054, 0.05169853570909567 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1007, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5035, "brier": 0.6012935021172318, "nll": 1.0622078369569694, "ece10": 0.05686138160772465, "calibration_n": 2000, "confident_wrong": 0 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "wilson95": [ 0.7735463743160095, 0.9801879358846942 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 26, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.15307254455437266, "nll": 0.3053698968853343, "ece10": 0.19315423165750514, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9282051282051282, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 6, "purchase": 1, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 1, "inspection": 0, "purchase": 0, "work": 6, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 24, "accuracy": 0.8571428571428571, "wilson95": [ 0.685101978364747, 0.9430099468025059 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 24, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.23261408139118606, "nll": 0.4193556104610673, "ece10": 0.1748073628202376, "calibration_n": 28, "confident_wrong": 1, "macro_f1": 0.8484848484848486, "confusion_matrix": { "work": { "work": 6, "purchase": 0, "inspection": 0, "design": 1, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 2, "purchase": 1, "inspection": 4, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 22, "accuracy": 0.7857142857142857, "wilson95": [ 0.6046141434627812, 0.8978753966710211 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.22797384226742728, "nll": 0.4300045339872408, "ece10": 0.15158562727904445, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.7836538461538461, "confusion_matrix": { "design": { "design": 4, "inspection": 2, "purchase": 0, "work": 1, "__failed__": 0 }, "inspection": { "design": 1, "inspection": 6, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 1, "inspection": 1, "purchase": 0, "work": 5, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 22, "accuracy": 0.7857142857142857, "wilson95": [ 0.6046141434627812, 0.8978753966710211 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.28517469008603213, "nll": 0.5391536106685553, "ece10": 0.23073438121694917, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.7854449472096531, "confusion_matrix": { "work": { "work": 6, "purchase": 0, "inspection": 0, "design": 1, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 2, "purchase": 0, "inspection": 5, "design": 0, "__failed__": 0 }, "design": { "work": 2, "purchase": 0, "inspection": 1, "design": 4, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 33, "accuracy": 0.825, "wilson95": [ 0.6805000966821229, 0.9125458625396408 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 33, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.825, "brier": 0.2300196312848623, "nll": 0.4943599637845903, "ece10": 0.2727096221458486, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8155228758169935, "confusion_matrix": { "review": { "review": 4, "it": 3, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 6, "documents": 1, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 1, "maintenance": 7, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 34, "accuracy": 0.85, "wilson95": [ 0.709276756335103, 0.9293881228267964 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.2556401703456379, "nll": 0.5324256730351233, "ece10": 0.22938588680638228, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8403594771241831, "confusion_matrix": { "maintenance": { "maintenance": 7, "procurement": 1, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 7, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 0, "documents": 1, "it": 1, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 30, "accuracy": 0.75, "wilson95": [ 0.5980603857923197, 0.858128813609037 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 30, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3830914230705781, "nll": 0.7567861452045769, "ece10": 0.1480509100292115, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7422222222222222, "confusion_matrix": { "review": { "review": 4, "it": 2, "documents": 0, "procurement": 1, "maintenance": 1, "__failed__": 0 }, "it": { "review": 0, "it": 5, "documents": 1, "procurement": 0, "maintenance": 2, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 6, "procurement": 2, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 1, "maintenance": 7, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 26, "accuracy": 0.65, "wilson95": [ 0.49505880837257693, 0.7786547112682372 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 26, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.39522845045482113, "nll": 0.7856428547199574, "ece10": 0.19528599049071055, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.6470607287449393, "confusion_matrix": { "maintenance": { "maintenance": 7, "procurement": 1, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 1, "procurement": 7, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 2, "documents": 5, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 3, "procurement": 0, "documents": 0, "it": 5, "review": 0, "__failed__": 0 }, "review": { "maintenance": 5, "procurement": 1, "documents": 0, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 222, "accuracy": 0.555, "wilson95": [ 0.506004433664882, 0.6029492139583456 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 222, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.555, "brier": 0.6260487686528624, "nll": 1.298406997685168, "ece10": 0.14439534842953944, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.5104911008979843, "confusion_matrix": { "sadness": { "sadness": 88, "joy": 5, "love": 18, "anger": 6, "fear": 2, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 20, "joy": 53, "love": 40, "anger": 3, "fear": 2, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 8, "joy": 3, "love": 23, "anger": 2, "fear": 1, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 22, "joy": 3, "love": 3, "anger": 35, "fear": 0, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 19, "joy": 2, "love": 4, "anger": 5, "fear": 20, "surprise": 2, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 3, "love": 1, "anger": 1, "fear": 0, "surprise": 3, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 246, "accuracy": 0.82, "wilson95": [ 0.7725668011129554, 0.859341697303335 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 246, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.82, "brier": 0.31745651535133945, "nll": 0.8529310257517131, "ece10": 0.23935148261178618, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 366, "accuracy": 0.915, "wilson95": [ 0.8835678157264721, 0.9385369799760619 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 366, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.915, "brier": 0.25790868676258105, "nll": 0.43845324944009056, "ece10": 0.24494496831224954, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.8751423010539459, "confusion_matrix": { "false": { "false": 296, "true": 13, "__failed__": 0 }, "true": { "false": 21, "true": 70, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 284, "accuracy": 0.71, "wilson95": [ 0.6637014956653163, 0.7523033407142791 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 284, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.71, "brier": 0.4865235898504809, "nll": 1.079566914057072, "ece10": 0.13529959696686517, "calibration_n": 400, "confident_wrong": 28, "macro_f1": 0.6683897130537446, "confusion_matrix": { "world": { "world": 91, "sports": 5, "business": 7, "sci_tech": 0, "__failed__": 0 }, "sports": { "world": 11, "sports": 112, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 15, "sports": 0, "business": 55, "sci_tech": 2, "__failed__": 0 }, "sci_tech": { "world": 33, "sports": 2, "business": 41, "sci_tech": 26, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 258, "accuracy": 0.645, "wilson95": [ 0.5969304612084719, 0.6903109734345821 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 258, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.645, "brier": 0.4357933722092757, "nll": 0.6262603501995619, "ece10": 0.04144625262026406, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.5063273536364901, "confusion_matrix": { "true": { "true": 23, "false": 130, "__failed__": 0 }, "false": { "true": 12, "false": 235, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 187, "accuracy": 0.6233333333333333, "wilson95": [ 0.5672684321681708, 0.6762796349297745 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 187, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6233333333333333, "brier": 0.5260849721490272, "nll": 1.3563643895774902, "ece10": 0.19083251556836664, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 275, "accuracy": 0.6875, "wilson95": [ 0.6404744504377174, 0.7309584391869214 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 275, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.4174736735434765, "nll": 0.6071251428560193, "ece10": 0.08378346204083317, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.6727213222058583, "confusion_matrix": { "false": { "false": 180, "true": 8, "__failed__": 0 }, "true": { "false": 117, "true": 95, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 262, "accuracy": 0.655, "wilson95": [ 0.6071390167491942, 0.6999121720071738 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 262, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.655, "brier": 0.43000166592778805, "nll": 0.620497265854968, "ece10": 0.06910722593816579, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.6278718584834431, "confusion_matrix": { "false": { "false": 185, "true": 15, "__failed__": 0 }, "true": { "false": 123, "true": 77, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 81, "accuracy": 0.81, "wilson95": [ 0.7222115462093562, 0.8748524849023126 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 81, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.81, "brier": 0.3214542400356271, "nll": 0.496212425890657, "ece10": 0.10798793236002203, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8067338012409724, "confusion_matrix": { "yes": { "yes": 47, "no": 7, "__failed__": 0 }, "no": { "yes": 12, "no": 34, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 64, "accuracy": 0.64, "wilson95": [ 0.5423540245160874, 0.7272877959859567 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 64, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.64, "brier": 0.4910904005927005, "nll": 1.1124022610265654, "ece10": 0.1261872480269084, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.6421085858585858, "confusion_matrix": { "balance": { "balance": 4, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 4, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 1, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 3, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 6, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 3, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 5, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 3, "branch_appointment": 2, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 6, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 1, "statement": 1, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 8, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 5, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 5, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 5, "other": 0, "__failed__": 0 }, "other": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 1, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 57, "accuracy": 0.57, "wilson95": [ 0.47215389549212305, 0.6626670147588988 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 151, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.755, "brier": 0.37956868648348135, "nll": 0.6515086772933527, "ece10": 0.15506790861215827, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 42, "accuracy": 0.42, "wilson95": [ 0.3279838267435473, 0.5179351329695703 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 128, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.64, "brier": 0.48847655022868997, "nll": 0.9784187513913737, "ece10": 0.06604269430436482, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.35000736177774205, "nll": 0.8839824466148061, "ece10": 0.17432291096938235, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.7675222993237698, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 14, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 15, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 1, "bank_statement": 0, "clinical_note": 5, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 2, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 2, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 7, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 2, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 6, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 5, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 17, "accuracy": 0.17, "wilson95": [ 0.10893566114059931, 0.25548004767601123 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 232, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.5250596420448456, "nll": 0.9748672355269464, "ece10": 0.08446934909709465, "calibration_n": 400, "confident_wrong": 0 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.5134358513092555, "nll": 1.1027652541833255, "ece10": 0.10972362005714484, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5823590660547182, "confusion_matrix": { "politics": { "politics": 2, "business": 1, "technology": 2, "science": 3, "health": 3, "sports": 0, "entertainment": 0, "world": 2, "culture": 1, "weather": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 10, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 2, "technology": 6, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 1, "business": 1, "technology": 0, "science": 6, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 7, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 2, "business": 0, "technology": 0, "science": 2, "health": 1, "sports": 3, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 2, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "world": { "politics": 1, "business": 0, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 7, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 0, "world": 2, "culture": 7, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 2, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 5, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 42, "accuracy": 0.42, "wilson95": [ 0.3279838267435473, 0.5179351329695703 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 42, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42, "brier": 0.7654606819569086, "nll": 1.8488681594468195, "ece10": 0.07386570375299613, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.4182807314416223, "confusion_matrix": { "biology": { "biology": 8, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 2, "medicine": 8, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 6, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 6, "medicine": 1, "chemistry": 1, "physics": 4, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 3, "medicine": 2, "chemistry": 1, "physics": 0, "computer_science": 4, "mathematics": 0, "earth_science": 2, "engineering": 3, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 3, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 2, "medicine": 3, "chemistry": 1, "physics": 1, "computer_science": 0, "mathematics": 0, "earth_science": 3, "engineering": 0, "economics": 1, "other": 0, "__failed__": 0 }, "engineering": { "biology": 1, "medicine": 1, "chemistry": 2, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 2, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 4, "other": 0, "__failed__": 0 }, "other": { "biology": 5, "medicine": 2, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 3, "economics": 1, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 18, "accuracy": 0.18, "wilson95": [ 0.1170022549894635, 0.2666735838630073 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 94, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47, "brier": 0.4205719644040856, "nll": 0.8226605142615702, "ece10": 0.12716454021101245, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 87, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.435, "brier": 0.33423814834188403, "nll": 0.5934299297406025, "ece10": 0.08722044245291821, "calibration_n": 100, "confident_wrong": 4 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 84, "accuracy": 0.84, "wilson95": [ 0.7557973061072976, 0.8990471151119521 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 84, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.84, "brier": 0.2303450848567008, "nll": 0.4194510553557007, "ece10": 0.08662514052068401, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.8386999677500269, "confusion_matrix": { "negative": { "negative": 36, "neutral": 0, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 11, "neutral": 21, "positive": 1, "__failed__": 0 }, "positive": { "negative": 3, "neutral": 1, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43, "brier": 0.2871936450853611, "nll": 0.46833790904133515, "ece10": 0.07862042428158705, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 40, "accuracy": 0.4, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 224, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7466666666666667, "brier": 0.37328701158237027, "nll": 0.7184786134845412, "ece10": 0.09224616455773747, "calibration_n": 300, "confident_wrong": 1 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 69, "accuracy": 0.69, "wilson95": [ 0.5937394428051057, 0.7722030278762395 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.69, "brier": 0.5077253381776835, "nll": 1.3234117547090065, "ece10": 0.11928447750023302, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6287079983508556, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 1, "password_reset": 5, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 5, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 2, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 3, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 3, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 4, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 1, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 3, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 3, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 9, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 5, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 4, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 2, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.7090947455875716, "nll": 1.6655522836454375, "ece10": 0.084511566504802, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.4559818585553879, "confusion_matrix": { "billing": { "billing": 2, "account_access": 2, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 2, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 1, "account_access": 8, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 3, "returns": 1, "technical_outage": 2, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 1, "__failed__": 0 }, "returns": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 7, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 1, "account_access": 1, "shipping": 1, "returns": 0, "technical_outage": 3, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 2, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 5, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 6, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 5, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "compliance": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 2, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 4, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 2, "account_access": 0, "shipping": 1, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 1, "data_privacy": 2, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 14, "accuracy": 0.14, "wilson95": [ 0.08526343659939134, 0.22137188210963837 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 146, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4866666666666667, "brier": 0.6015973384684755, "nll": 1.2231844909580791, "ece10": 0.05895389748713616, "calibration_n": 300, "confident_wrong": 0 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 63, "accuracy": 0.63, "wilson95": [ 0.5322052958094291, 0.7181763946567546 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 63, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.63, "brier": 0.4744043342747102, "nll": 1.0579805295991114, "ece10": 0.09947456049712794, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6113657733162378, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 4, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 2, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 4, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 4, "hotel": 0, "car": 2, "visa_docs": 0, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 4, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 8, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 1, "visa_docs": 0, "loyalty": 2, "disruption": 3, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 4, "car": 2, "visa_docs": 0, "loyalty": 1, "disruption": 2, "other": 2, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 71, "accuracy": 0.71, "wilson95": [ 0.6146110882442888, 0.7898516425087772 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 71, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.71, "brier": 0.46757633345353755, "nll": 0.8735338327948335, "ece10": 0.11886789005656218, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.6537687166774627, "confusion_matrix": { "business": { "business": 22, "science and technology": 0, "sports": 0, "world": 3, "__failed__": 0 }, "science and technology": { "business": 15, "science and technology": 3, "sports": 1, "world": 6, "__failed__": 0 }, "sports": { "business": 2, "science and technology": 0, "sports": 21, "world": 2, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 77, "accuracy": 0.77, "wilson95": [ 0.6784561697712622, 0.8415673411969654 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 77, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.77, "brier": 0.44818744486741463, "nll": 1.240771843491276, "ece10": 0.23926416632860467, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.7361772486772487, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 2, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 2, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 1, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 1, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 1, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 2, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 2, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 1, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 1, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 52, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.6439831002504763, "nll": 1.330215672869881, "ece10": 0.12476030614960365, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5125395940613332, "confusion_matrix": { "anger": { "anger": 10, "fear": 1, "joy": 0, "love": 1, "sadness": 4, "surprise": 1, "__failed__": 0 }, "fear": { "anger": 1, "fear": 10, "joy": 0, "love": 0, "sadness": 5, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 6, "love": 8, "sadness": 1, "surprise": 1, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 1, "love": 12, "sadness": 4, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 2, "fear": 4, "joy": 0, "love": 0, "sadness": 10, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 0, "joy": 2, "love": 4, "sadness": 5, "surprise": 4, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2067, "accuracy": 0.7011533242876526, "wilson95": [ 0.6843762332852709, 0.717406862638121 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2067, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7011533242876526, "brier": 0.44849948423780706, "nll": 1.0652182509126633, "ece10": 0.11172612549403636, "calibration_n": 2948, "confident_wrong": 16, "macro_f1": 0.70702877901452, "confusion_matrix": { "alarm": { "alarm": 94, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 2, "audio": 42, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 11, "iot": 0, "lists": 0, "music": 3, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 39, "audio": 0, "calendar": 262, "cooking": 12, "datetime": 22, "email": 1, "general": 10, "iot": 1, "lists": 9, "music": 4, "news": 9, "play": 4, "qa": 10, "recommendation": 13, "social": 3, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 67, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 2, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 91, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 5, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 2, "email": 222, "general": 25, "iot": 0, "lists": 4, "music": 0, "news": 1, "play": 0, "qa": 3, "recommendation": 2, "social": 8, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 3, "calendar": 6, "cooking": 8, "datetime": 2, "email": 0, "general": 74, "iot": 5, "lists": 2, "music": 3, "news": 4, "play": 4, "qa": 46, "recommendation": 18, "social": 6, "takeaway": 1, "transport": 4, "weather": 1, "__failed__": 0 }, "iot": { "alarm": 5, "audio": 14, "calendar": 0, "cooking": 21, "datetime": 0, "email": 0, "general": 14, "iot": 154, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 4, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 4, "datetime": 0, "email": 0, "general": 8, "iot": 1, "lists": 119, "music": 4, "news": 0, "play": 0, "qa": 2, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 1, "music": 75, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 112, "play": 1, "qa": 7, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "play": { "alarm": 2, "audio": 51, "calendar": 0, "cooking": 1, "datetime": 0, "email": 0, "general": 26, "iot": 0, "lists": 1, "music": 176, "news": 0, "play": 126, "qa": 1, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 2, "datetime": 5, "email": 0, "general": 14, "iot": 4, "lists": 4, "music": 2, "news": 4, "play": 6, "qa": 233, "recommendation": 2, "social": 4, "takeaway": 0, "transport": 4, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 3, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 3, "music": 2, "news": 11, "play": 15, "qa": 4, "recommendation": 53, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 13, "iot": 3, "lists": 1, "music": 0, "news": 2, "play": 0, "qa": 0, "recommendation": 1, "social": 84, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 10, "datetime": 1, "email": 0, "general": 0, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 43, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 7, "social": 0, "takeaway": 0, "transport": 108, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 25, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 13, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 0, "weather": 108, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1557, "accuracy": 0.5281546811397557, "wilson95": [ 0.5101093203623563, 0.5461267622036494 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1557, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5281546811397557, "brier": 0.6326029690166738, "nll": 1.6615596322721757, "ece10": 0.03913253867642096, "calibration_n": 2948, "confident_wrong": 6, "macro_f1": 0.5172130497689206, "confusion_matrix": { "alarm": { "alarm": 90, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 2, "audio": 26, "calendar": 0, "cooking": 0, "datetime": 3, "email": 2, "general": 20, "iot": 0, "lists": 1, "music": 5, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 19, "audio": 0, "calendar": 238, "cooking": 12, "datetime": 32, "email": 2, "general": 41, "iot": 1, "lists": 14, "music": 3, "news": 7, "play": 1, "qa": 9, "recommendation": 8, "social": 4, "takeaway": 5, "transport": 1, "weather": 3, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 56, "datetime": 1, "email": 1, "general": 1, "iot": 0, "lists": 5, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "datetime": { "alarm": 1, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 61, "email": 0, "general": 9, "iot": 0, "lists": 2, "music": 0, "news": 1, "play": 0, "qa": 14, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 6, "__failed__": 0 }, "email": { "alarm": 1, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 1, "email": 219, "general": 20, "iot": 0, "lists": 6, "music": 0, "news": 1, "play": 0, "qa": 4, "recommendation": 1, "social": 10, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "general": { "alarm": 2, "audio": 4, "calendar": 8, "cooking": 19, "datetime": 3, "email": 1, "general": 62, "iot": 2, "lists": 7, "music": 5, "news": 8, "play": 5, "qa": 38, "recommendation": 11, "social": 3, "takeaway": 2, "transport": 6, "weather": 1, "__failed__": 0 }, "iot": { "alarm": 42, "audio": 24, "calendar": 3, "cooking": 36, "datetime": 0, "email": 1, "general": 37, "iot": 33, "lists": 24, "music": 2, "news": 0, "play": 1, "qa": 0, "recommendation": 1, "social": 6, "takeaway": 1, "transport": 0, "weather": 3, "__failed__": 0 }, "lists": { "alarm": 1, "audio": 21, "calendar": 3, "cooking": 1, "datetime": 1, "email": 0, "general": 6, "iot": 0, "lists": 99, "music": 2, "news": 0, "play": 2, "qa": 1, "recommendation": 2, "social": 1, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 1, "datetime": 0, "email": 1, "general": 8, "iot": 0, "lists": 1, "music": 64, "news": 0, "play": 1, "qa": 0, "recommendation": 1, "social": 1, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 0, "email": 1, "general": 9, "iot": 0, "lists": 3, "music": 0, "news": 95, "play": 0, "qa": 7, "recommendation": 2, "social": 1, "takeaway": 0, "transport": 0, "weather": 5, "__failed__": 0 }, "play": { "alarm": 7, "audio": 46, "calendar": 1, "cooking": 18, "datetime": 0, "email": 1, "general": 55, "iot": 0, "lists": 21, "music": 154, "news": 3, "play": 76, "qa": 0, "recommendation": 0, "social": 2, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 6, "audio": 4, "calendar": 6, "cooking": 5, "datetime": 6, "email": 2, "general": 34, "iot": 3, "lists": 11, "music": 2, "news": 12, "play": 4, "qa": 167, "recommendation": 11, "social": 2, "takeaway": 2, "transport": 4, "weather": 6, "__failed__": 0 }, "recommendation": { "alarm": 1, "audio": 2, "calendar": 6, "cooking": 6, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 3, "music": 4, "news": 7, "play": 12, "qa": 7, "recommendation": 35, "social": 1, "takeaway": 1, "transport": 4, "weather": 1, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 2, "datetime": 0, "email": 1, "general": 20, "iot": 2, "lists": 3, "music": 0, "news": 1, "play": 0, "qa": 1, "recommendation": 0, "social": 74, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 1, "audio": 0, "calendar": 1, "cooking": 15, "datetime": 1, "email": 1, "general": 4, "iot": 1, "lists": 3, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 30, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 10, "cooking": 1, "datetime": 18, "email": 0, "general": 2, "iot": 0, "lists": 4, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 6, "social": 0, "takeaway": 10, "transport": 66, "weather": 3, "__failed__": 0 }, "weather": { "alarm": 6, "audio": 0, "calendar": 5, "cooking": 10, "datetime": 8, "email": 1, "general": 19, "iot": 1, "lists": 4, "music": 0, "news": 4, "play": 0, "qa": 23, "recommendation": 1, "social": 0, "takeaway": 3, "transport": 1, "weather": 66, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 703, "accuracy": 0.5993179880647911, "wilson95": [ 0.5709945170058485, 0.6269930693878939 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 703, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5993179880647911, "cluster_bootstrap95": [ 0.5643853622530093, 0.6322338275399033 ], "independent_source_groups": 47, "brier": 0.5344770383345432, "nll": 0.9124933930058339, "ece10": 0.06239834163863794, "calibration_n": 1173, "confident_wrong": 6, "macro_f1": 0.5443389914232188, "confusion_matrix": { "negative": { "negative": 93, "neutral": 77, "positive": 12, "__failed__": 0 }, "neutral": { "negative": 106, "neutral": 450, "positive": 34, "__failed__": 0 }, "positive": { "negative": 86, "neutral": 155, "positive": 160, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "imajev-2b": { "label": "Imajev 2B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 168, "accuracy": 0.84, "wilson95": [ 0.7828592983793289, 0.8843258796841297 ], "valid": 196, "failures": 4, "coverage": 0.98, "heads": 200, "head_correct": 168, "native_tie_decisions_changed": 0, "abstained_records": 4, "truncated_records": 0, "head_accuracy": 0.84, "brier": 0.20968203803605776, "nll": 0.4028803131595419, "ece10": 0.04830140980079879, "calibration_n": 196, "confident_wrong": 6, "macro_f1": 0.8490346989918063, "confusion_matrix": { "world": { "world": 46, "sports": 0, "business": 0, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 43, "sci_tech": 3, "__failed__": 3 }, "sci_tech": { "world": 5, "sports": 0, "business": 13, "sci_tech": 33, "__failed__": 1 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 121, "accuracy": 0.605, "wilson95": [ 0.5358834915928536, 0.6701589898679203 ], "valid": 191, "failures": 9, "coverage": 0.955, "heads": 200, "head_correct": 121, "native_tie_decisions_changed": 0, "abstained_records": 9, "truncated_records": 0, "head_accuracy": 0.605, "brier": 0.556277685346733, "nll": 1.1353966182517146, "ece10": 0.13312042307080998, "calibration_n": 191, "confident_wrong": 10, "macro_f1": 0.4534109856030089, "confusion_matrix": { "sadness": { "sadness": 38, "joy": 3, "love": 2, "anger": 8, "fear": 5, "surprise": 0, "__failed__": 1 }, "joy": { "sadness": 11, "joy": 54, "love": 4, "anger": 3, "fear": 2, "surprise": 0, "__failed__": 5 }, "love": { "sadness": 2, "joy": 7, "love": 3, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 5, "joy": 4, "love": 0, "anger": 15, "fear": 2, "surprise": 0, "__failed__": 2 }, "fear": { "sadness": 3, "joy": 2, "love": 1, "anger": 1, "fear": 11, "surprise": 1, "__failed__": 1 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 136, "accuracy": 0.68, "wilson95": [ 0.6124810156588646, 0.7407346668453195 ], "valid": 185, "failures": 15, "coverage": 0.925, "heads": 200, "head_correct": 136, "native_tie_decisions_changed": 0, "abstained_records": 15, "truncated_records": 0, "head_accuracy": 0.68, "brier": 0.32458569209954186, "nll": 0.4850524096712784, "ece10": 0.077207140485134, "calibration_n": 185, "confident_wrong": 0, "macro_f1": 0.6489434644393193, "confusion_matrix": { "false": { "false": 106, "true": 1, "__failed__": 2 }, "true": { "false": 48, "true": 30, "__failed__": 13 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 144, "accuracy": 0.72, "wilson95": [ 0.6540769541334929, 0.7776311022605097 ], "valid": 178, "failures": 22, "coverage": 0.89, "heads": 200, "head_correct": 144, "native_tie_decisions_changed": 0, "abstained_records": 22, "truncated_records": 0, "head_accuracy": 0.72, "brier": 0.28144810698845657, "nll": 0.44864901707311455, "ece10": 0.07347741538581548, "calibration_n": 178, "confident_wrong": 0, "macro_f1": 0.7436114732724902, "confusion_matrix": { "true": { "true": 41, "false": 34, "__failed__": 2 }, "false": { "true": 0, "false": 103, "__failed__": 20 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3539517609205183, "nll": 0.7164971654620552, "ece10": 0.24592491535099925, "calibration_n": 12, "confident_wrong": 1, "macro_f1": 0.7733333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.511051645546099, "nll": 1.0263099395175497, "ece10": 0.2785198597579452, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.6942857142857142, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 1, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 1, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04553928685619069, "nll": 0.14142139508092555, "ece10": 0.12631374721053187, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06547947206562231, "nll": 0.183610729780584, "ece10": 0.16014116114372612, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 83, "accuracy": 0.83, "wilson95": [ 0.7445199523239887, 0.8910643388594006 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 83, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.83, "brier": 0.2572359512961006, "nll": 0.4647317750778612, "ece10": 0.06853458021898702, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.7994045451350954, "confusion_matrix": { "label_000": { "label_000": 25, "label_001": 0, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 12, "label_001": 8, "label_002": 1, "label_003": 4, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 45, "accuracy": 0.45, "wilson95": [ 0.3561453797951198, 0.5475539700255787 ], "valid": 86, "failures": 14, "coverage": 0.86, "heads": 100, "head_correct": 45, "native_tie_decisions_changed": 0, "abstained_records": 14, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.6105742723102147, "nll": 1.254619752632257, "ece10": 0.1567660772005984, "calibration_n": 86, "confident_wrong": 1, "macro_f1": 0.4710320550499915, "confusion_matrix": { "label_000": { "label_000": 9, "label_001": 3, "label_002": 0, "label_003": 0, "label_004": 4, "label_005": 0, "__failed__": 1 }, "label_001": { "label_000": 0, "label_001": 11, "label_002": 0, "label_003": 0, "label_004": 6, "label_005": 0, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 7, "label_003": 3, "label_004": 1, "label_005": 1, "__failed__": 4 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 3, "label_003": 6, "label_004": 6, "label_005": 0, "__failed__": 2 }, "label_004": { "label_000": 1, "label_001": 1, "label_002": 2, "label_003": 1, "label_004": 10, "label_005": 0, "__failed__": 1 }, "label_005": { "label_000": 1, "label_001": 0, "label_002": 3, "label_003": 0, "label_004": 4, "label_005": 2, "__failed__": 6 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 77, "accuracy": 0.77, "wilson95": [ 0.6784561697712622, 0.8415673411969654 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 77, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.77, "brier": 0.6524168891871781, "nll": 1.8501214625634341, "ece10": 0.5275087926273793, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.7255291005291005, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 2, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 2, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 1, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 2, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 2, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 47, "accuracy": 0.9791666666666666, "wilson95": [ 0.8910078200474814, 0.9963128890224893 ], "valid": 47, "failures": 1, "coverage": 0.9791666666666666, "heads": 48, "head_correct": 47, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.9791666666666666, "brier": 0.0027079919241098497, "nll": 0.029288333535249622, "ece10": 0.028375770018678685, "calibration_n": 47, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 62, "accuracy": 0.5585585585585585, "wilson95": [ 0.4657611253397782, 0.6474384119483829 ], "valid": 109, "failures": 2, "coverage": 0.9819819819819819, "heads": 111, "head_correct": 62, "native_tie_decisions_changed": 0, "abstained_records": 2, "truncated_records": 0, "head_accuracy": 0.5585585585585585, "brier": 0.5085154984400322, "nll": 0.8679727374557985, "ece10": 0.10572452919002642, "calibration_n": 109, "confident_wrong": 3 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 65, "accuracy": 0.9027777777777778, "wilson95": [ 0.8126495420487314, 0.9521036800263443 ], "valid": 71, "failures": 1, "coverage": 0.9861111111111112, "heads": 72, "head_correct": 65, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.9027777777777778, "cluster_bootstrap95": [ 0.8194444444444444, 0.9722222222222222 ], "independent_source_groups": 36, "brier": 0.15284443041400717, "nll": 0.2881656656119633, "ece10": 0.0930756819778584, "calibration_n": 71, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 16, "accuracy": 0.8421052631578947, "wilson95": [ 0.6243452472967093, 0.9447952836146585 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8421052631578947, "brier": 0.22462347583758946, "nll": 0.36516910968719557, "ece10": 0.20568373959771083, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8347826086956522, "confusion_matrix": { "false": { "false": 10, "true": 1, "__failed__": 0 }, "true": { "false": 2, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.13597470591496508, "nll": 0.23759280105047692, "ece10": 0.08941449980532726, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 18, "accuracy": 0.8181818181818182, "wilson95": [ 0.6148339256287345, 0.9269311541657794 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8181818181818182, "brier": 0.24640529583323706, "nll": 0.39824648020052, "ece10": 0.0932035336581645, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8166666666666667, "confusion_matrix": { "false": { "false": 8, "true": 2, "__failed__": 0 }, "true": { "false": 2, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.12104806624731239, "nll": 0.2182883747322927, "ece10": 0.10499374320413446, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9415584415584415, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 3, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.37474137270540675, "nll": 0.6920017153606711, "ece10": 0.09248729573442356, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7319480519480519, "confusion_matrix": { "0": { "0": 1, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 3, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.011146377677638656, "nll": 0.062315330959304104, "ece10": 0.05819383847303159, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 14, "failures": 3, "coverage": 0.8235294117647058, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 3, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.18807499804796554, "nll": 0.37048543896552255, "ece10": 0.2379803011340933, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.7555555555555556, "confusion_matrix": { "bug": { "bug": 2, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 1 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 2, "refactor": 1, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 1, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 1, "__failed__": 2 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.1872296718821825, "nll": 0.37441887687113784, "ece10": 0.15375728375024553, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8539682539682539, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 2, "test": 0, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 1, "docs": 0, "test": 0, "chore": 3, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 16, "failures": 1, "coverage": 0.9411764705882353, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.18355698795686948, "nll": 0.3341550061132334, "ece10": 0.22527271618207423, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9080882352941176, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 1 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.29392126119557466, "nll": 0.46342632619872864, "ece10": 0.1513822809872118, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8117647058823529, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.4332340371305532, "nll": 0.7958855369759934, "ece10": 0.1584583057066647, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5244444444444444, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 1, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 1, "none": 3, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 6, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.21433582950537844, "nll": 0.4428837492646267, "ece10": 0.1872669136558005, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8555555555555555, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 4, "press_release": 0, "analysis": 1, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 2, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 2, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03267852834645874, "nll": 0.0858384857116875, "ece10": 0.07436968055066233, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.023373811446832808, "nll": 0.09890628363671515, "ece10": 0.0885824613393629, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.3109405185687341, "nll": 0.4646452527269081, "ece10": 0.16125232094493558, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6886446886446886, "confusion_matrix": { "false": { "false": 8, "true": 1, "__failed__": 0 }, "true": { "false": 4, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03659892155249509, "nll": 0.09034114503400135, "ece10": 0.07614486986862769, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.20654627070801268, "nll": 0.424758265976417, "ece10": 0.11644373426781919, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8628571428571428, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 2, "thanks": 0, "fyi": 1, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.11999951656770468, "nll": 0.2442047163093014, "ece10": 0.1770546215739885, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8711111111111112, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 4, "office_supplies": 0, "software": 0, "client_entertainment": 1, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.23796886014276833, "nll": 0.39337518788018844, "ece10": 0.1765604619360722, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8875, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.5672029565364528, "nll": 0.9126492300406498, "ece10": 0.1289411784816929, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5551948051948052, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 10, "accuracy": 0.5555555555555556, "wilson95": [ 0.33716415642042774, 0.7544048187299437 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5555555555555556, "brier": 0.4323599180985991, "nll": 0.6759990367271148, "ece10": 0.1912336437039677, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.4898989898989899, "confusion_matrix": { "0": { "0": 1, "1": 4, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 4, "2": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 16, "accuracy": 0.8, "wilson95": [ 0.5839825677481064, 0.919342337420202 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.28505286553616405, "nll": 0.5696810946038731, "ece10": 0.18351984779881414, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8083116883116883, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 1, "harassment": 0, "boosting": 0, "smurfing": 2, "other": 0, "__failed__": 0 }, "other": { "cheating": 1, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 4, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 6, "accuracy": 0.2857142857142857, "wilson95": [ 0.1381386295212213, 0.49956380716085597 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2857142857142857, "brier": 0.9275105418395173, "nll": 1.721759215757444, "ece10": 0.3841489682103886, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.18, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 0, "punctuation": 0, "none": 2, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.2211920533463918, "nll": 0.3579022987028903, "ece10": 0.14308504413335527, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8117647058823529, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.027591405508686856, "nll": 0.11379624958672233, "ece10": 0.10335416471062953, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 11, "accuracy": 0.7857142857142857, "wilson95": [ 0.524107694133997, 0.9242861328730683 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.47014725319684, "nll": 0.9161494651997327, "ece10": 0.3101586631539761, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.6476190476190476, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 2, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.34377584510378256, "nll": 0.6787242349674522, "ece10": 0.23937735117519954, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7749999999999999, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 1, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.518906672125596, "nll": 0.8937151178815907, "ece10": 0.21502846049580793, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5476190476190476, "confusion_matrix": { "0": { "0": 1, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 16, "failures": 1, "coverage": 0.9411764705882353, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.39783682253731795, "nll": 0.5845813491058003, "ece10": 0.1374980688617213, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6818181818181819, "confusion_matrix": { "false": { "false": 3, "true": 4, "__failed__": 1 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.11089493473841582, "nll": 0.21832898101933387, "ece10": 0.09744400638843498, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9377777777777778, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "sports": 2, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.026330731821689463, "nll": 0.10838164333585577, "ece10": 0.09851678203530068, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.38816008775737076, "nll": 0.5552337925134586, "ece10": 0.13242376234358152, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6886446886446886, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.15946053676884905, "nll": 0.3083558765127394, "ece10": 0.1894983437209533, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9372549019607843, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.2972290975058004, "nll": 0.5188885165267865, "ece10": 0.2106292449349681, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6793650793650793, "confusion_matrix": { "0": { "0": 2, "1": 3, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.12030893273191487, "nll": 0.299059975660362, "ece10": 0.15713645631163797, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571427, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 5, "accuracy": 0.29411764705882354, "wilson95": [ 0.1327998961029523, 0.5313311006653985 ], "valid": 10, "failures": 7, "coverage": 0.5882352941176471, "heads": 17, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 7, "truncated_records": 0, "head_accuracy": 0.29411764705882354, "brier": 0.4726609070183919, "nll": 0.6577471878884531, "ece10": 0.27242947915632587, "calibration_n": 10, "confident_wrong": 0, "macro_f1": 0.3125, "confusion_matrix": { "false": { "false": 5, "true": 2, "__failed__": 1 }, "true": { "false": 3, "true": 0, "__failed__": 6 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 16, "failures": 2, "coverage": 0.8888888888888888, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 2, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.11727224180609727, "nll": 0.2350731947215257, "ece10": 0.10426192441061707, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8833333333333332, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 1 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 0, "changed_mind": 2, "late_delivery": 0, "__failed__": 1 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 7, "accuracy": 0.3888888888888889, "wilson95": [ 0.2030524657216362, 0.613809583977621 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3888888888888889, "brier": 0.6742754204165737, "nll": 1.3696018511576549, "ece10": 0.2054411829044755, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.32564102564102565, "confusion_matrix": { "0": { "0": 0, "1": 2, "2": 1, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 0, "2": 3, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 2, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 1, "3": 1, "4": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.3013359222939876, "nll": 0.46889623465508234, "ece10": 0.2133467216840232, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6862745098039216, "confusion_matrix": { "false": { "false": 5, "true": 5, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 8, "accuracy": 0.5333333333333333, "wilson95": [ 0.30116980025498397, 0.7519046463426261 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5333333333333333, "brier": 0.6218426577510723, "nll": 0.8477589663534286, "ece10": 0.35221932842540016, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.4444444444444444, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.09810090978696863, "nll": 0.24069440247826257, "ece10": 0.18708001808246702, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9047619047619048, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 4, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 1, "sales": 0, "account": 0, "other": 1, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.25072411651825083, "nll": 0.4234353245702533, "ece10": 0.2471057228657658, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9411764705882353, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 12, "accuracy": 0.631578947368421, "wilson95": [ 0.41039533164803066, 0.8085048725486492 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.631578947368421, "brier": 0.45638769192388884, "nll": 0.7780484685851704, "ece10": 0.17386750830160175, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.5161782661782661, "confusion_matrix": { "0": { "0": 5, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 13, "failures": 4, "coverage": 0.7647058823529411, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 4, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.347100256136638, "nll": 0.5287760349093111, "ece10": 0.18671788894754343, "calibration_n": 13, "confident_wrong": 0, "macro_f1": 0.5341614906832298, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 2 }, "true": { "false": 3, "true": 1, "__failed__": 2 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.027392398291953883, "nll": 0.10687242253739096, "ece10": 0.09911593865993795, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 15, "accuracy": 0.75, "wilson95": [ 0.531299122381256, 0.8881382985923343 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.42148561218023967, "nll": 0.7723963286001092, "ece10": 0.23180881208330487, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7582251082251082, "confusion_matrix": { "0": { "0": 4, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 19, "failures": 1, "coverage": 0.95, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.010558839280790605, "nll": 0.06363745305325894, "ece10": 0.05951686444131331, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9714285714285713, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 3, "__failed__": 1 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 9, "accuracy": 0.45, "wilson95": [ 0.2581978582556243, 0.6579146575496577 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.5527176168005246, "nll": 0.7400924602642368, "ece10": 0.4078035872666904, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.3103448275862069, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 11, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 22, "accuracy": 1.0, "wilson95": [ 0.8513451253913791, 1.0 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.2582950418633552, "nll": 0.5281121225282918, "ece10": 0.395395018583763, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 6, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 498, "accuracy": 0.5533333333333333, "wilson95": [ 0.5206952602661087, 0.5855180574160189 ], "valid": 895, "failures": 5, "coverage": 0.9944444444444445, "heads": 1073, "head_correct": 653, "native_tie_decisions_changed": 0, "abstained_records": 5, "truncated_records": 0, "head_accuracy": 0.608574091332712, "cluster_bootstrap95": [ 0.5233292682926829, 0.5829113172081145 ], "independent_source_groups": 596, "brier": 0.49522571945618155, "nll": 0.861168445802794, "ece10": 0.14194701099437892, "calibration_n": 1068, "confident_wrong": 25 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 345, "accuracy": 0.6010452961672473, "wilson95": [ 0.5604415953206378, 0.6403055096668483 ], "valid": 529, "failures": 45, "coverage": 0.921602787456446, "heads": 936, "head_correct": 682, "native_tie_decisions_changed": 0, "abstained_records": 44, "truncated_records": 0, "head_accuracy": 0.7286324786324786, "brier": 0.3373208464077116, "nll": 0.681322195559792, "ece10": 0.035922871618100594, "calibration_n": 890, "confident_wrong": 23 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 222, "accuracy": 0.3171428571428571, "wilson95": [ 0.2837466032618096, 0.3525351235507384 ], "valid": 669, "failures": 31, "coverage": 0.9557142857142857, "heads": 1088, "head_correct": 470, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4319852941176471, "cluster_bootstrap95": [ 0.28364688856729375, 0.35135328185328185 ], "independent_source_groups": 650, "brier": 0.6476152466991769, "nll": 1.2200158372767222, "ece10": 0.1177808531569685, "calibration_n": 1026, "confident_wrong": 9 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 37, "accuracy": 0.0925, "wilson95": [ 0.06785393449857381, 0.12489858538387784 ], "valid": 304, "failures": 96, "coverage": 0.76, "heads": 2000, "head_correct": 1252, "native_tie_decisions_changed": 0, "abstained_records": 96, "truncated_records": 0, "head_accuracy": 0.626, "brier": 0.44152571680707076, "nll": 0.7712019560446215, "ece10": 0.034655804692663356, "calibration_n": 1849, "confident_wrong": 6 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.022166355310694646, "nll": 0.09533679396215673, "ece10": 0.08723672377071132, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.050728834449865694, "nll": 0.14896659942012494, "ece10": 0.12976814134922793, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.06811653668065283, "nll": 0.17287654397888488, "ece10": 0.08715835802566708, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 24, "accuracy": 0.8571428571428571, "wilson95": [ 0.685101978364747, 0.9430099468025059 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 24, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.1669324871302731, "nll": 0.31203769277543564, "ece10": 0.14969802861395737, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.8565233785822021, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 1, "purchase": 5, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 2, "purchase": 0, "inspection": 0, "design": 5, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 36, "accuracy": 0.9, "wilson95": [ 0.7694822477247771, 0.9604204713173936 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 36, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.155804033834894, "nll": 0.33132055181525855, "ece10": 0.1376901979801463, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8875816993464053, "confusion_matrix": { "review": { "review": 4, "it": 0, "documents": 1, "procurement": 1, "maintenance": 2, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 36, "accuracy": 0.9, "wilson95": [ 0.7694822477247771, 0.9604204713173936 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 36, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.1405122891399874, "nll": 0.3036153241035546, "ece10": 0.13005071394399748, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8862745098039216, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 1, "documents": 1, "it": 1, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 32, "accuracy": 0.8, "wilson95": [ 0.652426936536005, 0.895000102745623 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 32, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.31987472541671813, "nll": 0.6542139735958153, "ece10": 0.18370149504267322, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7911661506707945, "confusion_matrix": { "review": { "review": 4, "it": 1, "documents": 1, "procurement": 1, "maintenance": 1, "__failed__": 0 }, "it": { "review": 0, "it": 6, "documents": 0, "procurement": 0, "maintenance": 2, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 7, "procurement": 1, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 1, "procurement": 7, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 30, "accuracy": 0.75, "wilson95": [ 0.5980603857923197, 0.858128813609037 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 30, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.32657312160079766, "nll": 0.6511789512628853, "ece10": 0.1162845390129484, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7337407690348867, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 7, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 1, "documents": 7, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 3, "procurement": 0, "documents": 0, "it": 5, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 1, "documents": 1, "it": 1, "review": 3, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 239, "accuracy": 0.5975, "wilson95": [ 0.5487342037489487, 0.6444108988558637 ], "valid": 366, "failures": 34, "coverage": 0.915, "heads": 400, "head_correct": 239, "native_tie_decisions_changed": 0, "abstained_records": 34, "truncated_records": 0, "head_accuracy": 0.5975, "brier": 0.49520234969932514, "nll": 1.0045663006120111, "ece10": 0.06942168391300095, "calibration_n": 366, "confident_wrong": 4, "macro_f1": 0.5078998226109592, "confusion_matrix": { "sadness": { "sadness": 83, "joy": 11, "love": 4, "anger": 8, "fear": 9, "surprise": 1, "__failed__": 3 }, "joy": { "sadness": 8, "joy": 81, "love": 7, "anger": 8, "fear": 2, "surprise": 3, "__failed__": 9 }, "love": { "sadness": 8, "joy": 12, "love": 8, "anger": 3, "fear": 2, "surprise": 0, "__failed__": 4 }, "anger": { "sadness": 11, "joy": 1, "love": 1, "anger": 36, "fear": 2, "surprise": 5, "__failed__": 8 }, "fear": { "sadness": 8, "joy": 2, "love": 0, "anger": 4, "fear": 29, "surprise": 1, "__failed__": 8 }, "surprise": { "sadness": 1, "joy": 2, "love": 1, "anger": 1, "fear": 1, "surprise": 2, "__failed__": 2 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 278, "accuracy": 0.9266666666666666, "wilson95": [ 0.8914686583942826, 0.9510760061607713 ], "valid": 287, "failures": 13, "coverage": 0.9566666666666667, "heads": 300, "head_correct": 278, "native_tie_decisions_changed": 0, "abstained_records": 13, "truncated_records": 0, "head_accuracy": 0.9266666666666666, "brier": 0.08788041235789894, "nll": 0.29275922504879553, "ece10": 0.1713376112484892, "calibration_n": 287, "confident_wrong": 0 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 273, "accuracy": 0.6825, "wilson95": [ 0.6353297856503707, 0.7261982269176112 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 273, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6825, "brier": 0.45650128046972854, "nll": 0.7258923884609668, "ece10": 0.16136688513361117, "calibration_n": 400, "confident_wrong": 38, "macro_f1": 0.6570325211485359, "confusion_matrix": { "false": { "false": 191, "true": 118, "__failed__": 0 }, "true": { "false": 9, "true": 82, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 326, "accuracy": 0.815, "wilson95": [ 0.7740142283989209, 0.8499930261704722 ], "valid": 393, "failures": 7, "coverage": 0.9825, "heads": 400, "head_correct": 326, "native_tie_decisions_changed": 0, "abstained_records": 7, "truncated_records": 0, "head_accuracy": 0.815, "brier": 0.26829297051091944, "nll": 0.5076115578624022, "ece10": 0.08331387035409377, "calibration_n": 393, "confident_wrong": 21, "macro_f1": 0.806627552467385, "confusion_matrix": { "world": { "world": 88, "sports": 6, "business": 8, "sci_tech": 0, "__failed__": 1 }, "sports": { "world": 6, "sports": 117, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 5, "sports": 0, "business": 64, "sci_tech": 1, "__failed__": 2 }, "sci_tech": { "world": 13, "sports": 0, "business": 28, "sci_tech": 57, "__failed__": 4 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 296, "accuracy": 0.74, "wilson95": [ 0.6948757457668063, 0.7805583529527312 ], "valid": 356, "failures": 44, "coverage": 0.89, "heads": 400, "head_correct": 296, "native_tie_decisions_changed": 0, "abstained_records": 44, "truncated_records": 0, "head_accuracy": 0.74, "brier": 0.27683781668045676, "nll": 0.4465506586239305, "ece10": 0.08954887298770686, "calibration_n": 356, "confident_wrong": 0, "macro_f1": 0.7641039977634889, "confusion_matrix": { "true": { "true": 87, "false": 55, "__failed__": 11 }, "false": { "true": 5, "false": 209, "__failed__": 33 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 228, "accuracy": 0.76, "wilson95": [ 0.7085788119154636, 0.8048468430477725 ], "valid": 299, "failures": 1, "coverage": 0.9966666666666667, "heads": 300, "head_correct": 228, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.3463210390900634, "nll": 0.8358126903434588, "ece10": 0.12552097261218118, "calibration_n": 299, "confident_wrong": 0 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 277, "accuracy": 0.6925, "wilson95": [ 0.6456251644125997, 0.7357126022686962 ], "valid": 377, "failures": 23, "coverage": 0.9425, "heads": 400, "head_correct": 277, "native_tie_decisions_changed": 0, "abstained_records": 23, "truncated_records": 0, "head_accuracy": 0.6925, "brier": 0.3202066611105719, "nll": 0.4791241889533961, "ece10": 0.06527295636462578, "calibration_n": 377, "confident_wrong": 0, "macro_f1": 0.6985312422205626, "confusion_matrix": { "false": { "false": 180, "true": 0, "__failed__": 8 }, "true": { "false": 100, "true": 97, "__failed__": 15 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 305, "accuracy": 0.7625, "wilson95": [ 0.7184235572819022, 0.8015824881925919 ], "valid": 398, "failures": 2, "coverage": 0.995, "heads": 400, "head_correct": 305, "native_tie_decisions_changed": 0, "abstained_records": 2, "truncated_records": 0, "head_accuracy": 0.7625, "brier": 0.30421102509092385, "nll": 0.46897965807070374, "ece10": 0.05400468760715927, "calibration_n": 398, "confident_wrong": 0, "macro_f1": 0.7593460196271232, "confusion_matrix": { "false": { "false": 182, "true": 16, "__failed__": 2 }, "true": { "false": 77, "true": 123, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 68, "accuracy": 0.68, "wilson95": [ 0.5833738138954762, 0.7633085267500092 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 68, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.68, "brier": 0.41452766459684964, "nll": 0.6083652645973282, "ece10": 0.096646536911382, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.6259934548854604, "confusion_matrix": { "yes": { "yes": 53, "no": 1, "__failed__": 0 }, "no": { "yes": 31, "no": 15, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 59, "accuracy": 0.59, "wilson95": [ 0.4920143005341078, 0.6813268697886348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 59, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.59, "brier": 0.5334837020612442, "nll": 1.3042857387838507, "ece10": 0.056078931510771424, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5687339607855912, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 3, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 7, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 2, "statement": 3, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 6, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 6, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 2, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 10, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 3, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 3, "limit_increase": 0, "other": 1, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 3, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 55, "accuracy": 0.55, "wilson95": [ 0.45244602997442135, 0.6438546202048803 ], "valid": 99, "failures": 1, "coverage": 0.99, "heads": 200, "head_correct": 150, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.30302908609055196, "nll": 0.5440116831457665, "ece10": 0.09401343175745462, "calibration_n": 199, "confident_wrong": 1 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 131, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.655, "brier": 0.44950086309809906, "nll": 0.9163872610422424, "ece10": 0.08062958212694017, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 75, "accuracy": 0.75, "wilson95": [ 0.656955364519384, 0.8245478863771232 ], "valid": 93, "failures": 7, "coverage": 0.93, "heads": 100, "head_correct": 75, "native_tie_decisions_changed": 0, "abstained_records": 7, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.2976017527505554, "nll": 0.7062707603856073, "ece10": 0.08897950205735863, "calibration_n": 93, "confident_wrong": 0, "macro_f1": 0.7423614954811898, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 14, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 10, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 3 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 6, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 2 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 1 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 9, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 2, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 0, "__failed__": 1 }, "recipe": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 3, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 219, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5475, "brier": 0.6131079751240146, "nll": 1.1738327393578813, "ece10": 0.16431031852252725, "calibration_n": 400, "confident_wrong": 9 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 57, "accuracy": 0.57, "wilson95": [ 0.47215389549212305, 0.6626670147588988 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 57, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.57, "brier": 0.5518940089559797, "nll": 1.205134523569579, "ece10": 0.11366411458809286, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5525615715606566, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "science": 3, "health": 5, "sports": 0, "entertainment": 1, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 10, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 3, "technology": 7, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 1, "business": 1, "technology": 0, "science": 5, "health": 3, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 1, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 5, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 2, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 1, "weather": 0, "__failed__": 0 }, "world": { "politics": 3, "business": 4, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 1, "business": 0, "technology": 0, "science": 1, "health": 1, "sports": 0, "entertainment": 2, "world": 0, "culture": 5, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 2, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 6, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 43, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43, "brier": 0.7385744765050943, "nll": 1.8219374363929837, "ece10": 0.10781923045412477, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.4041450152657049, "confusion_matrix": { "biology": { "biology": 8, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 8, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 2, "medicine": 0, "chemistry": 3, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 3, "medicine": 2, "chemistry": 0, "physics": 4, "computer_science": 0, "mathematics": 0, "earth_science": 3, "engineering": 3, "economics": 1, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 1, "medicine": 2, "chemistry": 1, "physics": 0, "computer_science": 4, "mathematics": 0, "earth_science": 1, "engineering": 5, "economics": 1, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 1, "chemistry": 1, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 1, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 2, "medicine": 4, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 5, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 5, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 0, "economics": 5, "other": 0, "__failed__": 0 }, "other": { "biology": 2, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 1, "mathematics": 1, "earth_science": 3, "engineering": 3, "economics": 1, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 10, "accuracy": 0.1, "wilson95": [ 0.0552291370606751, 0.17436566150491345 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 75, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.375, "brier": 0.5313759342010624, "nll": 1.0143736315827392, "ece10": 0.14081545887042773, "calibration_n": 100, "confident_wrong": 4 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 13, "accuracy": 0.13, "wilson95": [ 0.07757167427240512, 0.20980351440076428 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 89, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.445, "brier": 0.4529491850723798, "nll": 0.7370080589832522, "ece10": 0.14885830313744572, "calibration_n": 100, "confident_wrong": 2 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 84, "accuracy": 0.84, "wilson95": [ 0.7557973061072976, 0.8990471151119521 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 84, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.84, "brier": 0.2670617605588608, "nll": 0.48969435538683515, "ece10": 0.16028020003494045, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8365881169819526, "confusion_matrix": { "negative": { "negative": 33, "neutral": 3, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 3, "neutral": 24, "positive": 6, "__failed__": 0 }, "positive": { "negative": 1, "neutral": 3, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 19, "accuracy": 0.19, "wilson95": [ 0.12514751509768735, 0.27778845379064376 ], "valid": 99, "failures": 1, "coverage": 0.99, "heads": 200, "head_correct": 97, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.485, "brier": 0.2503945804915801, "nll": 0.38687554265450225, "ece10": 0.1386995087781375, "calibration_n": 99, "confident_wrong": 0 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 99, "failures": 1, "coverage": 0.99, "heads": 300, "head_correct": 228, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.3392070727410029, "nll": 0.6649403087816261, "ece10": 0.05899441942648336, "calibration_n": 299, "confident_wrong": 4 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 59, "accuracy": 0.59, "wilson95": [ 0.4920143005341078, 0.6813268697886348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 59, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.59, "brier": 0.5810712025273643, "nll": 1.6580641912370115, "ece10": 0.08707284894667132, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5377742975487336, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 1, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 2, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 1, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 3, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 2, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 4, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 3, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 2, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 2, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 3, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 9, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 4, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 2, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 2, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 2, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 40, "accuracy": 0.4, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.7844074454279393, "nll": 1.8269718162185022, "ece10": 0.2744431889556441, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.40213365539452495, "confusion_matrix": { "billing": { "billing": 5, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 3, "account_access": 5, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 4, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 3, "account_access": 1, "shipping": 1, "returns": 0, "technical_outage": 2, "onboarding": 0, "integrations": 3, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 1, "onboarding": 2, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 3, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 4, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 3, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 3, "pricing": 0, "data_privacy": 1, "other": 1, "__failed__": 0 }, "pricing": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 3, "data_privacy": 1, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 3, "account_access": 0, "shipping": 1, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 6, "accuracy": 0.06, "wilson95": [ 0.027786123963188192, 0.1247681544589592 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 135, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.6422689799746779, "nll": 1.32799768762859, "ece10": 0.1320349224318923, "calibration_n": 300, "confident_wrong": 0 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 63, "accuracy": 0.63, "wilson95": [ 0.5322052958094291, 0.7181763946567546 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 63, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.63, "brier": 0.5206311556361283, "nll": 1.1857886845070176, "ece10": 0.0924665251347931, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.576474063570838, "confusion_matrix": { "new_booking": { "new_booking": 0, "change_flight": 2, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 7, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 2, "cancel_trip": 0, "seat": 6, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 2, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 4, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 8, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 1, "disruption": 5, "other": 1, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 2, "refund": 1, "hotel": 4, "car": 1, "visa_docs": 1, "loyalty": 1, "disruption": 1, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 87, "accuracy": 0.87, "wilson95": [ 0.7901964855992357, 0.9224283257275948 ], "valid": 96, "failures": 4, "coverage": 0.96, "heads": 100, "head_correct": 87, "native_tie_decisions_changed": 0, "abstained_records": 4, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.15546011242105887, "nll": 0.30065284436644474, "ece10": 0.06540475995707278, "calibration_n": 96, "confident_wrong": 1, "macro_f1": 0.8809878314469419, "confusion_matrix": { "business": { "business": 23, "science and technology": 1, "sports": 0, "world": 0, "__failed__": 1 }, "science and technology": { "business": 6, "science and technology": 15, "sports": 1, "world": 1, "__failed__": 2 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 24, "__failed__": 1 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 83, "accuracy": 0.83, "wilson95": [ 0.7445199523239887, 0.8910643388594006 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 83, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.83, "brier": 0.4448018481973208, "nll": 1.240231605255424, "ece10": 0.43088551865302605, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.7912037037037036, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 2, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 2, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 2, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 2, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 2, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 1, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 46, "accuracy": 0.46, "wilson95": [ 0.36560808309024106, 0.5573513967663177 ], "valid": 88, "failures": 12, "coverage": 0.88, "heads": 100, "head_correct": 46, "native_tie_decisions_changed": 0, "abstained_records": 12, "truncated_records": 0, "head_accuracy": 0.46, "brier": 0.6645831631544542, "nll": 1.330666906024241, "ece10": 0.18724442842090594, "calibration_n": 88, "confident_wrong": 1, "macro_f1": 0.48378981060289683, "confusion_matrix": { "anger": { "anger": 8, "fear": 3, "joy": 0, "love": 0, "sadness": 4, "surprise": 0, "__failed__": 2 }, "fear": { "anger": 0, "fear": 12, "joy": 0, "love": 0, "sadness": 5, "surprise": 0, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 5, "love": 4, "sadness": 1, "surprise": 2, "__failed__": 4 }, "love": { "anger": 0, "fear": 0, "joy": 3, "love": 6, "sadness": 6, "surprise": 0, "__failed__": 2 }, "sadness": { "anger": 0, "fear": 1, "joy": 2, "love": 1, "sadness": 11, "surprise": 0, "__failed__": 1 }, "surprise": { "anger": 1, "fear": 0, "joy": 4, "love": 0, "sadness": 4, "surprise": 4, "__failed__": 3 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2439, "accuracy": 0.8273405698778833, "wilson95": [ 0.8132734334255909, 0.8405557192996589 ], "valid": 2886, "failures": 62, "coverage": 0.9789687924016283, "heads": 2948, "head_correct": 2439, "native_tie_decisions_changed": 0, "abstained_records": 62, "truncated_records": 0, "head_accuracy": 0.8273405698778833, "brier": 0.24097303001670822, "nll": 0.6030776322489566, "ece10": 0.10964087246501716, "calibration_n": 2886, "confident_wrong": 3, "macro_f1": 0.8437878060735114, "confusion_matrix": { "alarm": { "alarm": 90, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 53, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 1 }, "calendar": { "alarm": 2, "audio": 0, "calendar": 374, "cooking": 0, "datetime": 5, "email": 0, "general": 2, "iot": 0, "lists": 4, "music": 0, "news": 3, "play": 0, "qa": 1, "recommendation": 4, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 3 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 67, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 98, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 231, "general": 11, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 1, "social": 2, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 15 }, "general": { "alarm": 0, "audio": 0, "calendar": 11, "cooking": 5, "datetime": 0, "email": 0, "general": 114, "iot": 4, "lists": 0, "music": 1, "news": 9, "play": 1, "qa": 23, "recommendation": 9, "social": 1, "takeaway": 0, "transport": 2, "weather": 3, "__failed__": 4 }, "iot": { "alarm": 1, "audio": 4, "calendar": 0, "cooking": 2, "datetime": 0, "email": 0, "general": 6, "iot": 195, "lists": 0, "music": 1, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 4 }, "lists": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 2, "lists": 123, "music": 3, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 3 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 0, "lists": 2, "music": 73, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 115, "play": 0, "qa": 5, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 3 }, "play": { "alarm": 0, "audio": 12, "calendar": 0, "cooking": 0, "datetime": 2, "email": 2, "general": 7, "iot": 10, "lists": 1, "music": 138, "news": 4, "play": 195, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 12 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 22, "iot": 1, "lists": 0, "music": 1, "news": 3, "play": 1, "qa": 238, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 2, "weather": 3, "__failed__": 11 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 2, "play": 1, "qa": 2, "recommendation": 76, "social": 0, "takeaway": 0, "transport": 2, "weather": 0, "__failed__": 3 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 15, "iot": 1, "lists": 0, "music": 0, "news": 3, "play": 0, "qa": 0, "recommendation": 0, "social": 85, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 1 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 50, "transport": 0, "weather": 0, "__failed__": 2 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 116, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 3, "iot": 1, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 146, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2156, "accuracy": 0.7313432835820896, "wilson95": [ 0.7150489207650665, 0.7470355167050065 ], "valid": 2911, "failures": 37, "coverage": 0.987449118046133, "heads": 2948, "head_correct": 2156, "native_tie_decisions_changed": 0, "abstained_records": 37, "truncated_records": 0, "head_accuracy": 0.7313432835820896, "brier": 0.3654658575173884, "nll": 0.9198515073937036, "ece10": 0.08177029137026036, "calibration_n": 2911, "confident_wrong": 1, "macro_f1": 0.746387901455845, "confusion_matrix": { "alarm": { "alarm": 82, "audio": 1, "calendar": 4, "cooking": 0, "datetime": 4, "email": 0, "general": 0, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 2, "audio": 46, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 7, "iot": 1, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "calendar": { "alarm": 3, "audio": 1, "calendar": 343, "cooking": 0, "datetime": 15, "email": 1, "general": 13, "iot": 0, "lists": 2, "music": 0, "news": 7, "play": 0, "qa": 3, "recommendation": 5, "social": 1, "takeaway": 1, "transport": 1, "weather": 3, "__failed__": 1 }, "cooking": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 61, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 98, "email": 0, "general": 0, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 1, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 0, "email": 226, "general": 18, "iot": 0, "lists": 2, "music": 0, "news": 2, "play": 0, "qa": 3, "recommendation": 1, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 9 }, "general": { "alarm": 0, "audio": 2, "calendar": 14, "cooking": 4, "datetime": 1, "email": 0, "general": 105, "iot": 3, "lists": 1, "music": 1, "news": 15, "play": 1, "qa": 16, "recommendation": 10, "social": 0, "takeaway": 1, "transport": 6, "weather": 6, "__failed__": 1 }, "iot": { "alarm": 2, "audio": 19, "calendar": 0, "cooking": 1, "datetime": 0, "email": 3, "general": 14, "iot": 156, "lists": 2, "music": 0, "news": 1, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 6, "transport": 2, "weather": 4, "__failed__": 2 }, "lists": { "alarm": 0, "audio": 4, "calendar": 5, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 1, "lists": 116, "music": 3, "news": 1, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 1, "weather": 0, "__failed__": 3 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 3, "music": 69, "news": 1, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 1, "music": 1, "news": 109, "play": 0, "qa": 5, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 1 }, "play": { "alarm": 1, "audio": 22, "calendar": 3, "cooking": 1, "datetime": 5, "email": 0, "general": 29, "iot": 3, "lists": 5, "music": 173, "news": 10, "play": 112, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 3, "weather": 2, "__failed__": 11 }, "qa": { "alarm": 0, "audio": 3, "calendar": 0, "cooking": 1, "datetime": 1, "email": 0, "general": 48, "iot": 3, "lists": 0, "music": 2, "news": 11, "play": 1, "qa": 204, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 3, "weather": 4, "__failed__": 5 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 10, "cooking": 0, "datetime": 0, "email": 0, "general": 8, "iot": 0, "lists": 0, "music": 0, "news": 2, "play": 1, "qa": 1, "recommendation": 65, "social": 0, "takeaway": 1, "transport": 5, "weather": 0, "__failed__": 1 }, "social": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 4, "general": 16, "iot": 1, "lists": 0, "music": 0, "news": 6, "play": 0, "qa": 0, "recommendation": 0, "social": 74, "takeaway": 1, "transport": 0, "weather": 1, "__failed__": 1 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 48, "transport": 1, "weather": 0, "__failed__": 1 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 1, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 4, "social": 1, "takeaway": 0, "transport": 110, "weather": 2, "__failed__": 1 }, "weather": { "alarm": 0, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 0, "email": 1, "general": 5, "iot": 1, "lists": 0, "music": 1, "news": 3, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 132, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 770, "accuracy": 0.6564364876385337, "wilson95": [ 0.6287885928037311, 0.6830630990696742 ], "valid": 1149, "failures": 24, "coverage": 0.979539641943734, "heads": 1173, "head_correct": 770, "native_tie_decisions_changed": 0, "abstained_records": 24, "truncated_records": 0, "head_accuracy": 0.6564364876385337, "cluster_bootstrap95": [ 0.62188642687947, 0.6887023059855373 ], "independent_source_groups": 47, "brier": 0.4563925616954966, "nll": 0.7801117656410377, "ece10": 0.013962230052353577, "calibration_n": 1149, "confident_wrong": 5, "macro_f1": 0.5979088072454353, "confusion_matrix": { "negative": { "negative": 64, "neutral": 106, "positive": 12, "__failed__": 0 }, "neutral": { "negative": 29, "neutral": 504, "positive": 34, "__failed__": 23 }, "positive": { "negative": 14, "neutral": 184, "positive": 202, "__failed__": 1 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "imajev-4b": { "label": "Imajev 4B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 168, "accuracy": 0.84, "wilson95": [ 0.7828592983793289, 0.8843258796841297 ], "valid": 187, "failures": 13, "coverage": 0.935, "heads": 200, "head_correct": 168, "native_tie_decisions_changed": 0, "abstained_records": 13, "truncated_records": 0, "head_accuracy": 0.84, "brier": 0.16171361505350593, "nll": 0.31246957111180407, "ece10": 0.018758305585328282, "calibration_n": 187, "confident_wrong": 5, "macro_f1": 0.8709938603555625, "confusion_matrix": { "world": { "world": 42, "sports": 0, "business": 1, "sci_tech": 2, "__failed__": 5 }, "sports": { "world": 0, "sports": 44, "business": 2, "sci_tech": 0, "__failed__": 1 }, "business": { "world": 1, "sports": 0, "business": 44, "sci_tech": 2, "__failed__": 4 }, "sci_tech": { "world": 1, "sports": 0, "business": 10, "sci_tech": 38, "__failed__": 3 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 80, "accuracy": 0.4, "wilson95": [ 0.33460578566807336, 0.46916327960737997 ], "valid": 140, "failures": 60, "coverage": 0.7, "heads": 200, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 60, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.7125400091164588, "nll": 1.6240711156883145, "ece10": 0.2983166063594088, "calibration_n": 140, "confident_wrong": 25, "macro_f1": 0.3829342162675496, "confusion_matrix": { "sadness": { "sadness": 33, "joy": 1, "love": 2, "anger": 7, "fear": 4, "surprise": 0, "__failed__": 10 }, "joy": { "sadness": 13, "joy": 20, "love": 7, "anger": 2, "fear": 2, "surprise": 1, "__failed__": 34 }, "love": { "sadness": 2, "joy": 2, "love": 4, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 4 }, "anger": { "sadness": 2, "joy": 1, "love": 0, "anger": 14, "fear": 1, "surprise": 0, "__failed__": 10 }, "fear": { "sadness": 4, "joy": 2, "love": 1, "anger": 1, "fear": 9, "surprise": 1, "__failed__": 2 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 169, "accuracy": 0.845, "wilson95": [ 0.7883930940765792, 0.8886036307231068 ], "valid": 194, "failures": 6, "coverage": 0.97, "heads": 200, "head_correct": 169, "native_tie_decisions_changed": 0, "abstained_records": 6, "truncated_records": 0, "head_accuracy": 0.845, "brier": 0.191701143876049, "nll": 0.30768731375371766, "ece10": 0.030380520816572318, "calibration_n": 194, "confident_wrong": 3, "macro_f1": 0.8475967812092214, "confusion_matrix": { "false": { "false": 108, "true": 0, "__failed__": 1 }, "true": { "false": 25, "true": 61, "__failed__": 5 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 164, "accuracy": 0.82, "wilson95": [ 0.7608852497127511, 0.8670537414057983 ], "valid": 195, "failures": 5, "coverage": 0.975, "heads": 200, "head_correct": 164, "native_tie_decisions_changed": 0, "abstained_records": 5, "truncated_records": 0, "head_accuracy": 0.82, "brier": 0.22854788213282068, "nll": 0.35333212858981294, "ece10": 0.04794581520762293, "calibration_n": 195, "confident_wrong": 6, "macro_f1": 0.8093333333333333, "confusion_matrix": { "true": { "true": 47, "false": 30, "__failed__": 0 }, "false": { "true": 1, "false": 117, "__failed__": 5 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.2456791510829239, "nll": 0.4468698252190488, "ece10": 0.2286228772067227, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8533333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.2808709313043325, "nll": 0.5692659080069139, "ece10": 0.1858364696581408, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.76, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 1, "documents": 0, "it": 1, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00846861767717895, "nll": 0.06273808921005068, "ece10": 0.06007148657306349, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.008982303963102985, "nll": 0.0712214204417054, "ece10": 0.06822802917286808, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 86, "accuracy": 0.86, "wilson95": [ 0.7786281178903616, 0.9147365634006086 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.17856040127088985, "nll": 0.3130779668272418, "ece10": 0.08655937364313254, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8412217052546066, "confusion_matrix": { "label_000": { "label_000": 25, "label_001": 0, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 9, "label_001": 11, "label_002": 1, "label_003": 4, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 36, "accuracy": 0.36, "wilson95": [ 0.2727122040140433, 0.4576459754839126 ], "valid": 61, "failures": 39, "coverage": 0.61, "heads": 100, "head_correct": 36, "native_tie_decisions_changed": 0, "abstained_records": 39, "truncated_records": 0, "head_accuracy": 0.36, "brier": 0.6312804192966041, "nll": 1.3717914508054971, "ece10": 0.26340717866645424, "calibration_n": 61, "confident_wrong": 6, "macro_f1": 0.41241085806303196, "confusion_matrix": { "label_000": { "label_000": 6, "label_001": 4, "label_002": 0, "label_003": 1, "label_004": 1, "label_005": 0, "__failed__": 5 }, "label_001": { "label_000": 1, "label_001": 11, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "__failed__": 5 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 5, "label_003": 1, "label_004": 2, "label_005": 2, "__failed__": 7 }, "label_003": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 7, "label_004": 2, "label_005": 0, "__failed__": 7 }, "label_004": { "label_000": 1, "label_001": 2, "label_002": 0, "label_003": 1, "label_004": 7, "label_005": 0, "__failed__": 5 }, "label_005": { "label_000": 1, "label_001": 1, "label_002": 1, "label_003": 1, "label_004": 2, "label_005": 0, "__failed__": 10 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.1402958268221114, "nll": 0.2815529946286752, "ece10": 0.05850419137217869, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.8495370370370369, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 2, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 2, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 2, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 2, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 2, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0001879746125135798, "nll": 0.006226586012775769, "ece10": 0.006174178092264023, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 80, "accuracy": 0.7207207207207207, "wilson95": [ 0.630951748750832, 0.7957234302583911 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7207207207207207, "brier": 0.3674306716102128, "nll": 0.6305603837185562, "ece10": 0.08871533401499287, "calibration_n": 111, "confident_wrong": 0 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 70, "accuracy": 0.9722222222222222, "wilson95": [ 0.9042582477856177, 0.9923489780955054 ], "valid": 71, "failures": 1, "coverage": 0.9861111111111112, "heads": 72, "head_correct": 70, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.9722222222222222, "cluster_bootstrap95": [ 0.9305555555555556, 1.0 ], "independent_source_groups": 36, "brier": 0.02807385960878768, "nll": 0.08693048014578848, "ece10": 0.05964095744229748, "calibration_n": 71, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.09087001130891818, "nll": 0.1887502551100824, "ece10": 0.09972716295495586, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9467787114845938, "confusion_matrix": { "false": { "false": 10, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.014079316401827217, "nll": 0.06370550370843857, "ece10": 0.05968064491093524, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.1985718261530119, "nll": 0.33890592544122816, "ece10": 0.10689632064472028, "calibration_n": 22, "confident_wrong": 2, "macro_f1": 0.8562091503267973, "confusion_matrix": { "false": { "false": 7, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.05367832184621594, "nll": 0.08497084716626173, "ece10": 0.059128876114503494, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9415584415584416, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 1, "feature_request": 0, "praise": 0, "pricing_billing": 3, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.10689841949395452, "nll": 0.25360077281178833, "ece10": 0.20441325962361193, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.008032351954470742, "nll": 0.038249902417802864, "ece10": 0.03569837099571764, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 15, "failures": 2, "coverage": 0.8823529411764706, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 2, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.18240103873474062, "nll": 0.3352103208219294, "ece10": 0.13782430068473842, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8039682539682539, "confusion_matrix": { "bug": { "bug": 1, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 2 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 1, "security": 0, "style": 1, "refactor": 0, "nit": 2, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.12525634112699127, "nll": 0.2411446602855999, "ece10": 0.10991018391364887, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9259259259259259, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 0, "chore": 1, "__failed__": 0 }, "test": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 2, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.09906166071891924, "nll": 0.167219659962889, "ece10": 0.12462697296012093, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8785714285714286, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 15, "failures": 1, "coverage": 0.9375, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.21899449201205484, "nll": 0.38011146949934577, "ece10": 0.06178003092377874, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.829059829059829, "confusion_matrix": { "false": { "false": 5, "true": 2, "__failed__": 1 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 16, "failures": 2, "coverage": 0.8888888888888888, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 2, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.13871185244819056, "nll": 0.31841993157887, "ece10": 0.13530329326506907, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8444444444444444, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 1, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 4, "__failed__": 2 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.16101437214957084, "nll": 0.31538232523784687, "ece10": 0.1748721281864535, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 2, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 2, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.001723325696666359, "nll": 0.015113087241137423, "ece10": 0.014607520704826783, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00034743885510083133, "nll": 0.01341400820500493, "ece10": 0.013290833333642735, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.07570838995571504, "nll": 0.16762937628805502, "ece10": 0.1398380984709988, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9403508771929825, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0010118810976982087, "nll": 0.02087323509459576, "ece10": 0.020522625993630438, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.1261962700192473, "nll": 0.20485237250092359, "ece10": 0.1290733545628764, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.08899945653381554, "nll": 0.14970985080796323, "ece10": 0.10184931850196305, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8711111111111112, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 4, "office_supplies": 0, "software": 0, "client_entertainment": 1, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.16559056527385005, "nll": 0.26278413484084434, "ece10": 0.1226626162637547, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8875, "confusion_matrix": { "false": { "false": 7, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.4639119539472798, "nll": 0.7373264003960196, "ece10": 0.2294209318099052, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.4875, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.27908576103014965, "nll": 0.4523426854271466, "ece10": 0.22325079235413925, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7777777777777777, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 3, "2": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 17, "accuracy": 0.85, "wilson95": [ 0.6395811352592431, 0.9476312541037835 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.23014575813977584, "nll": 0.41361650391839067, "ece10": 0.10237599909703293, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8428571428571429, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 2, "boosting": 0, "smurfing": 0, "other": 2, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 1, "other": 0, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 3, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 9, "accuracy": 0.42857142857142855, "wilson95": [ 0.2446996892537596, 0.6345344563069328 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42857142857142855, "brier": 0.7026132930622808, "nll": 1.264453658708603, "ece10": 0.26059850970759246, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.38192982456140345, "confusion_matrix": { "spelling": { "spelling": 1, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 2, "none": 1, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 0, "punctuation": 0, "none": 1, "__failed__": 0 }, "word_choice": { "spelling": 1, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 3, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 1, "none": 4, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.014904482505222765, "nll": 0.07169384284840072, "ece10": 0.06750244347116699, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.007360845735972609, "nll": 0.045445928694526426, "ece10": 0.043153428860201126, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 11, "accuracy": 0.7857142857142857, "wilson95": [ 0.524107694133997, 0.9242861328730683 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.259038450681868, "nll": 0.47701262518559856, "ece10": 0.2323261600358017, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.6992063492063492, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.21864877202374378, "nll": 0.41326958559637994, "ece10": 0.16173981562398743, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8875, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.29628019530431327, "nll": 0.6450763433380697, "ece10": 0.2398775104164937, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8666666666666667, "confusion_matrix": { "0": { "0": 2, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.29037935527778597, "nll": 0.4229518838068385, "ece10": 0.13036843651019533, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7571428571428571, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.013740217779246411, "nll": 0.07648662903439764, "ece10": 0.07183064447365356, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 16, "failures": 1, "coverage": 0.9411764705882353, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.06562709897214082, "nll": 0.14271259833342295, "ece10": 0.07191016093956665, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9206349206349206, "confusion_matrix": { "payments": { "payments": 3, "web": 0, "infra": 1, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 3, "infra": 0, "data": 0, "model": 0, "__failed__": 1 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.3395368205410077, "nll": 0.48199566737848454, "ece10": 0.23835443162225303, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6640316205533596, "confusion_matrix": { "false": { "false": 3, "true": 5, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.018586614609187705, "nll": 0.08360509965564505, "ece10": 0.07839786598423353, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.15839637544222648, "nll": 0.31062188539874513, "ece10": 0.1761561271103106, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9393939393939394, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 17, "failures": 1, "coverage": 0.9444444444444444, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.047770473292012124, "nll": 0.08741145427846435, "ece10": 0.06363657820298868, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.902116402116402, "confusion_matrix": { "italian": { "italian": 1, "french": 1, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 1 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.31038275812891836, "nll": 0.44910482202243074, "ece10": 0.1796107768596613, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7017543859649122, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 4, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 17, "failures": 1, "coverage": 0.9444444444444444, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.11451343603372069, "nll": 0.18039874048803742, "ece10": 0.11683727446304523, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9214285714285715, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 1 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 0, "changed_mind": 3, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 11, "accuracy": 0.6111111111111112, "wilson95": [ 0.3861904160223791, 0.7969475342783638 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6111111111111112, "brier": 0.5068105169695025, "nll": 1.1239855295522299, "ece10": 0.13953728215078387, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5919047619047619, "confusion_matrix": { "0": { "0": 1, "1": 2, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 2, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.33108784232324456, "nll": 0.47765472435939244, "ece10": 0.17804855820684434, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6862745098039216, "confusion_matrix": { "false": { "false": 5, "true": 5, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 12, "accuracy": 0.8, "wilson95": [ 0.5481455128483065, 0.9295245065301844 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.33016284062412343, "nll": 0.5046859410916621, "ece10": 0.14540100439595735, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.7963800904977376, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.16308583513431923, "nll": 0.3202387647012208, "ece10": 0.14484133217836043, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.9199999999999999, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 4, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.10090558503660309, "nll": 0.20550018041004892, "ece10": 0.16720241537546698, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9411764705882353, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 12, "accuracy": 0.631578947368421, "wilson95": [ 0.41039533164803066, 0.8085048725486492 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.631578947368421, "brier": 0.4338404567977929, "nll": 0.6889816525969897, "ece10": 0.1280143940995066, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.48429144385026734, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 3, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 2, "2": 0, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 16, "failures": 1, "coverage": 0.9411764705882353, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.07622358522979625, "nll": 0.17336204751304687, "ece10": 0.14705061096127434, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9115384615384616, "confusion_matrix": { "false": { "false": 9, "true": 1, "__failed__": 1 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.0463687336856739, "nll": 0.10418440757627927, "ece10": 0.08771134403002648, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 14, "accuracy": 0.7, "wilson95": [ 0.4810271816464765, 0.8545227551323956 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.3392274284903479, "nll": 0.5754649941494528, "ece10": 0.12880054131468766, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.65, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.060166300810736484, "nll": 0.09066933604122662, "ece10": 0.053297397668631015, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 15, "accuracy": 0.75, "wilson95": [ 0.531299122381256, 0.8881382985923343 ], "valid": 19, "failures": 1, "coverage": 0.95, "heads": 20, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3432184607264693, "nll": 0.5343721391222866, "ece10": 0.14055578439155747, "calibration_n": 19, "confident_wrong": 2, "macro_f1": 0.7620320855614974, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 6, "__failed__": 1 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 21, "accuracy": 0.9545454545454546, "wilson95": [ 0.7820195416134267, 0.991930572378736 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9545454545454546, "brier": 0.14975245852013994, "nll": 0.36518338176897414, "ece10": 0.18878604147104724, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9580419580419581, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 553, "accuracy": 0.6144444444444445, "wilson95": [ 0.582223092016255, 0.6456929855099771 ], "valid": 863, "failures": 37, "coverage": 0.9588888888888889, "heads": 1073, "head_correct": 715, "native_tie_decisions_changed": 0, "abstained_records": 37, "truncated_records": 0, "head_accuracy": 0.6663560111835974, "cluster_bootstrap95": [ 0.575903113493475, 0.6601144049307298 ], "independent_source_groups": 596, "brier": 0.432487577707399, "nll": 0.7852435872796094, "ece10": 0.15035626860092416, "calibration_n": 1036, "confident_wrong": 68 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 353, "accuracy": 0.6149825783972126, "wilson95": [ 0.5745359632867671, 0.6539003975610619 ], "valid": 455, "failures": 119, "coverage": 0.7926829268292683, "heads": 936, "head_correct": 678, "native_tie_decisions_changed": 0, "abstained_records": 118, "truncated_records": 0, "head_accuracy": 0.7243589743589743, "brier": 0.2164664680246043, "nll": 0.4371616004635739, "ece10": 0.021837722083314214, "calibration_n": 799, "confident_wrong": 21 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 409, "accuracy": 0.5842857142857143, "wilson95": [ 0.5474127816301366, 0.6202386099172046 ], "valid": 668, "failures": 32, "coverage": 0.9542857142857143, "heads": 1088, "head_correct": 719, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.6608455882352942, "cluster_bootstrap95": [ 0.5490711341383642, 0.6187766714082503 ], "independent_source_groups": 650, "brier": 0.3945658232979107, "nll": 0.7432589747774802, "ece10": 0.02792014197072433, "calibration_n": 1025, "confident_wrong": 8 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 72, "accuracy": 0.18, "wilson95": [ 0.14545031532583302, 0.22063755304811672 ], "valid": 387, "failures": 13, "coverage": 0.9675, "heads": 2000, "head_correct": 1363, "native_tie_decisions_changed": 0, "abstained_records": 13, "truncated_records": 0, "head_accuracy": 0.6815, "brier": 0.41696880408433473, "nll": 0.727415478821373, "ece10": 0.016119987401928222, "calibration_n": 1986, "confident_wrong": 17 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.005932106411488861, "nll": 0.04978307406242451, "ece10": 0.04781952139694105, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.010772445402014422, "nll": 0.058214592972228654, "ece10": 0.05465606413282725, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.044003393939441526, "nll": 0.12505262966899774, "ece10": 0.10837197764474639, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0368360890781902, "nll": 0.12303623170408289, "ece10": 0.10973304058657803, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.09019446313972021, "nll": 0.21814550516641487, "ece10": 0.11842397933116537, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9499019607843138, "confusion_matrix": { "review": { "review": 7, "it": 0, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 7, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.06197761109399012, "nll": 0.17340426160553152, "ece10": 0.11369025420537954, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9499019607843138, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 7, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 34, "accuracy": 0.85, "wilson95": [ 0.709276756335103, 0.9293881228267964 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.17901004792156142, "nll": 0.3685045013037599, "ece10": 0.12320042963457639, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8487481146304676, "confusion_matrix": { "review": { "review": 5, "it": 0, "documents": 1, "procurement": 1, "maintenance": 1, "__failed__": 0 }, "it": { "review": 0, "it": 7, "documents": 0, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 6, "procurement": 0, "maintenance": 2, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 37, "accuracy": 0.925, "wilson95": [ 0.8013576647568946, 0.9741639742254119 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.1549113736613651, "nll": 0.33760273428748644, "ece10": 0.17362214522166589, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9228991596638656, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 7, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 1, "it": 0, "review": 6, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 169, "accuracy": 0.4225, "wilson95": [ 0.3750554237475196, 0.47141898187429637 ], "valid": 270, "failures": 130, "coverage": 0.675, "heads": 400, "head_correct": 169, "native_tie_decisions_changed": 0, "abstained_records": 130, "truncated_records": 0, "head_accuracy": 0.4225, "brier": 0.589214853611292, "nll": 1.308132366289972, "ece10": 0.22450911569559334, "calibration_n": 270, "confident_wrong": 28, "macro_f1": 0.4346624790779668, "confusion_matrix": { "sadness": { "sadness": 66, "joy": 4, "love": 5, "anger": 8, "fear": 8, "surprise": 0, "__failed__": 28 }, "joy": { "sadness": 8, "joy": 29, "love": 18, "anger": 3, "fear": 2, "surprise": 2, "__failed__": 56 }, "love": { "sadness": 8, "joy": 4, "love": 14, "anger": 4, "fear": 2, "surprise": 1, "__failed__": 4 }, "anger": { "sadness": 5, "joy": 0, "love": 3, "anger": 33, "fear": 0, "surprise": 2, "__failed__": 21 }, "fear": { "sadness": 3, "joy": 1, "love": 1, "anger": 4, "fear": 26, "surprise": 1, "__failed__": 16 }, "surprise": { "sadness": 1, "joy": 0, "love": 1, "anger": 1, "fear": 1, "surprise": 1, "__failed__": 5 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 277, "accuracy": 0.9233333333333333, "wilson95": [ 0.8875898961498251, 0.9483723882133924 ], "valid": 284, "failures": 16, "coverage": 0.9466666666666667, "heads": 300, "head_correct": 277, "native_tie_decisions_changed": 0, "abstained_records": 16, "truncated_records": 0, "head_accuracy": 0.9233333333333333, "brier": 0.03711801530001219, "nll": 0.08465271332981551, "ece10": 0.026085305376005, "calibration_n": 284, "confident_wrong": 2 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 307, "accuracy": 0.7675, "wilson95": [ 0.7236773296522552, 0.8062335928788962 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 307, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7675, "brier": 0.33480380635132084, "nll": 0.4884487752108941, "ece10": 0.13229211269624203, "calibration_n": 400, "confident_wrong": 21, "macro_f1": 0.7355682084746125, "confusion_matrix": { "false": { "false": 223, "true": 86, "__failed__": 0 }, "true": { "false": 7, "true": 84, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 320, "accuracy": 0.8, "wilson95": [ 0.758029683133299, 0.8362629402661231 ], "valid": 379, "failures": 21, "coverage": 0.9475, "heads": 400, "head_correct": 320, "native_tie_decisions_changed": 0, "abstained_records": 21, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.2392601953998054, "nll": 0.43821632811499883, "ece10": 0.07616812005970944, "calibration_n": 379, "confident_wrong": 25, "macro_f1": 0.8045616183417328, "confusion_matrix": { "world": { "world": 80, "sports": 6, "business": 10, "sci_tech": 0, "__failed__": 7 }, "sports": { "world": 2, "sports": 119, "business": 0, "sci_tech": 0, "__failed__": 2 }, "business": { "world": 2, "sports": 0, "business": 66, "sci_tech": 1, "__failed__": 3 }, "sci_tech": { "world": 11, "sports": 0, "business": 27, "sci_tech": 55, "__failed__": 9 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 335, "accuracy": 0.8375, "wilson95": [ 0.7981666097704109, 0.870412591553939 ], "valid": 381, "failures": 19, "coverage": 0.9525, "heads": 400, "head_correct": 335, "native_tie_decisions_changed": 0, "abstained_records": 19, "truncated_records": 0, "head_accuracy": 0.8375, "brier": 0.18159253332139996, "nll": 0.29794522287756603, "ece10": 0.020510889982315456, "calibration_n": 381, "confident_wrong": 9, "macro_f1": 0.8382440476190476, "confusion_matrix": { "true": { "true": 100, "false": 43, "__failed__": 10 }, "false": { "true": 3, "false": 235, "__failed__": 9 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 237, "accuracy": 0.79, "wilson95": [ 0.7403888774230657, 0.8322781992666975 ], "valid": 292, "failures": 8, "coverage": 0.9733333333333334, "heads": 300, "head_correct": 237, "native_tie_decisions_changed": 0, "abstained_records": 8, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.24439514160733408, "nll": 0.5516548908081643, "ece10": 0.09420613134911868, "calibration_n": 292, "confident_wrong": 0 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 333, "accuracy": 0.8325, "wilson95": [ 0.7927798983351753, 0.8658944259325176 ], "valid": 386, "failures": 14, "coverage": 0.965, "heads": 400, "head_correct": 333, "native_tie_decisions_changed": 0, "abstained_records": 14, "truncated_records": 0, "head_accuracy": 0.8325, "brier": 0.19359909481609588, "nll": 0.3150772508508959, "ece10": 0.019746861098929636, "calibration_n": 386, "confident_wrong": 7, "macro_f1": 0.8464923701890527, "confusion_matrix": { "false": { "false": 181, "true": 0, "__failed__": 7 }, "true": { "false": 53, "true": 152, "__failed__": 7 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 257, "accuracy": 0.6425, "wilson95": [ 0.5943817784800669, 0.6879072176346586 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 257, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6425, "brier": 0.5410034684459571, "nll": 0.8506401290212636, "ece10": 0.2205463689696444, "calibration_n": 400, "confident_wrong": 60, "macro_f1": 0.602421613806813, "confusion_matrix": { "false": { "false": 192, "true": 8, "__failed__": 0 }, "true": { "false": 135, "true": 65, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 69, "accuracy": 0.69, "wilson95": [ 0.5937394428051057, 0.7722030278762395 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.69, "brier": 0.405642320706693, "nll": 0.5997463859477289, "ece10": 0.12935152401330652, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.6693333333333333, "confusion_matrix": { "yes": { "yes": 47, "no": 7, "__failed__": 0 }, "no": { "yes": 24, "no": 22, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 53, "accuracy": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 53, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.53, "brier": 0.7114936059104376, "nll": 1.7744417550951386, "ece10": 0.2496815656742553, "calibration_n": 100, "confident_wrong": 8, "macro_f1": 0.5549591727716728, "confusion_matrix": { "balance": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 5, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 4, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 3, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 6, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 8, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 4, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 4, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 2, "limit_increase": 0, "other": 2, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 4, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 3, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 22, "accuracy": 0.22, "wilson95": [ 0.150012823846054, 0.31070353514985793 ], "valid": 37, "failures": 63, "coverage": 0.37, "heads": 200, "head_correct": 112, "native_tie_decisions_changed": 0, "abstained_records": 63, "truncated_records": 0, "head_accuracy": 0.56, "brier": 0.2485468641821117, "nll": 0.5159075394622163, "ece10": 0.0675900964504852, "calibration_n": 137, "confident_wrong": 1 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 97, "failures": 3, "coverage": 0.97, "heads": 200, "head_correct": 130, "native_tie_decisions_changed": 0, "abstained_records": 3, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.4639190804852376, "nll": 0.9511868011491543, "ece10": 0.08558632311608966, "calibration_n": 197, "confident_wrong": 4 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 79, "accuracy": 0.79, "wilson95": [ 0.7002003116591013, 0.858343459380847 ], "valid": 89, "failures": 11, "coverage": 0.89, "heads": 100, "head_correct": 79, "native_tie_decisions_changed": 0, "abstained_records": 11, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.18117346388028135, "nll": 0.4594695280679279, "ece10": 0.02008695571742163, "calibration_n": 89, "confident_wrong": 2, "macro_f1": 0.784858312279712, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 13, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 1 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 12, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 3 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 6, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 4 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 2, "recipe": 0, "sports_recap": 0, "__failed__": 1 }, "lease": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 6, "recipe": 0, "sports_recap": 0, "__failed__": 1 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 6, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 1, "sports_recap": 6, "__failed__": 1 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 6, "accuracy": 0.06, "wilson95": [ 0.027786123963188192, 0.1247681544589592 ], "valid": 82, "failures": 18, "coverage": 0.82, "heads": 400, "head_correct": 210, "native_tie_decisions_changed": 0, "abstained_records": 18, "truncated_records": 0, "head_accuracy": 0.525, "brier": 0.7274872630197818, "nll": 1.498759175170507, "ece10": 0.28448594425894, "calibration_n": 382, "confident_wrong": 54 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 61, "accuracy": 0.61, "wilson95": [ 0.5120301704584073, 0.6998312599360558 ], "valid": 92, "failures": 8, "coverage": 0.92, "heads": 100, "head_correct": 61, "native_tie_decisions_changed": 0, "abstained_records": 8, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.5228138270801297, "nll": 1.2134561922064875, "ece10": 0.09153183985688788, "calibration_n": 92, "confident_wrong": 1, "macro_f1": 0.6299723668144721, "confusion_matrix": { "politics": { "politics": 5, "business": 0, "technology": 0, "science": 2, "health": 3, "sports": 0, "entertainment": 0, "world": 0, "culture": 1, "weather": 1, "__failed__": 2 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 1 }, "technology": { "politics": 0, "business": 2, "technology": 7, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 1 }, "science": { "politics": 0, "business": 2, "technology": 1, "science": 6, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 1, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 5, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 4, "world": 0, "culture": 2, "weather": 1, "__failed__": 0 }, "world": { "politics": 1, "business": 1, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 3, "culture": 1, "weather": 0, "__failed__": 2 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 7, "weather": 1, "__failed__": 1 }, "weather": { "politics": 0, "business": 2, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 5, "__failed__": 1 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 41, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.41, "brier": 0.7845968431008095, "nll": 1.8748510380034376, "ece10": 0.2091451786519962, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.37778183502321433, "confusion_matrix": { "biology": { "biology": 7, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 9, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 1, "medicine": 0, "chemistry": 4, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 3, "medicine": 3, "chemistry": 1, "physics": 5, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 2, "economics": 1, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 3, "medicine": 2, "chemistry": 2, "physics": 0, "computer_science": 2, "mathematics": 0, "earth_science": 2, "engineering": 3, "economics": 1, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 1, "physics": 2, "computer_science": 1, "mathematics": 1, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 2, "medicine": 3, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 0, "earth_science": 5, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 2, "earth_science": 0, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 0, "economics": 4, "other": 0, "__failed__": 0 }, "other": { "biology": 2, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 2, "earth_science": 3, "engineering": 3, "economics": 1, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 6, "accuracy": 0.06, "wilson95": [ 0.027786123963188192, 0.1247681544589592 ], "valid": 95, "failures": 5, "coverage": 0.95, "heads": 200, "head_correct": 70, "native_tie_decisions_changed": 0, "abstained_records": 5, "truncated_records": 0, "head_accuracy": 0.35, "brier": 0.5727348442529641, "nll": 1.1981233226943702, "ece10": 0.23582535789846662, "calibration_n": 95, "confident_wrong": 14 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 29, "accuracy": 0.29, "wilson95": [ 0.2101483574912227, 0.3853889117557111 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 119, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.595, "brier": 0.34160595640283875, "nll": 0.5759114207440656, "ece10": 0.11349410893626331, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 86, "accuracy": 0.86, "wilson95": [ 0.7786281178903616, 0.9147365634006086 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.24087607467263364, "nll": 0.4459858118208336, "ece10": 0.1716929191768795, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8576537317562426, "confusion_matrix": { "negative": { "negative": 35, "neutral": 1, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 7, "neutral": 24, "positive": 2, "__failed__": 0 }, "positive": { "negative": 1, "neutral": 3, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 22, "accuracy": 0.22, "wilson95": [ 0.150012823846054, 0.31070353514985793 ], "valid": 85, "failures": 15, "coverage": 0.85, "heads": 200, "head_correct": 103, "native_tie_decisions_changed": 0, "abstained_records": 15, "truncated_records": 0, "head_accuracy": 0.515, "brier": 0.18661257151287663, "nll": 0.29916351387725126, "ece10": 0.0496442300651421, "calibration_n": 85, "confident_wrong": 1 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 44, "accuracy": 0.44, "wilson95": [ 0.34672026630160796, 0.5377189534832302 ], "valid": 97, "failures": 3, "coverage": 0.97, "heads": 300, "head_correct": 231, "native_tie_decisions_changed": 0, "abstained_records": 3, "truncated_records": 0, "head_accuracy": 0.77, "brier": 0.3366295750833437, "nll": 0.6868972583309801, "ece10": 0.07061646611105193, "calibration_n": 297, "confident_wrong": 15 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 61, "accuracy": 0.61, "wilson95": [ 0.5120301704584073, 0.6998312599360558 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 61, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.5103651888905752, "nll": 1.463974363885411, "ece10": 0.119043768277576, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5818900606494593, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 3, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 5, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 3, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 3, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 2, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 4, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "speak_to_human": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 2, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 3, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 4, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 4, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 2, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 1, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 1, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 2, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 40, "accuracy": 0.4, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.8325744126856108, "nll": 2.0353529471002063, "ece10": 0.29168187117985633, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.3835036513958083, "confusion_matrix": { "billing": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 2, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 4, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 3, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 3, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 7, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 3, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 2, "onboarding": 0, "integrations": 3, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 1, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 2, "integrations": 4, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 8, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 3, "pricing": 0, "data_privacy": 1, "other": 2, "__failed__": 0 }, "pricing": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 1, "account_access": 0, "shipping": 1, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 9, "accuracy": 0.09, "wilson95": [ 0.04807254000256516, 0.1622621285271631 ], "valid": 99, "failures": 1, "coverage": 0.99, "heads": 300, "head_correct": 151, "native_tie_decisions_changed": 0, "abstained_records": 1, "truncated_records": 0, "head_accuracy": 0.5033333333333333, "brier": 0.6750653758615026, "nll": 1.5685937556860656, "ece10": 0.21998228278638154, "calibration_n": 299, "confident_wrong": 10 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.642166816036918, "nll": 1.6211430242025906, "ece10": 0.23313850015832324, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.5320101827996564, "confusion_matrix": { "new_booking": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 4, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 2, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 2, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 11, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 2, "loyalty": 1, "disruption": 0, "other": 1, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 4, "disruption": 2, "other": 1, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 7, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 4, "car": 2, "visa_docs": 1, "loyalty": 2, "disruption": 1, "other": 1, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 86, "accuracy": 0.86, "wilson95": [ 0.7786281178903616, 0.9147365634006086 ], "valid": 98, "failures": 2, "coverage": 0.98, "heads": 100, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 2, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.1652183381735023, "nll": 0.2953842101093307, "ece10": 0.07173195596241896, "calibration_n": 98, "confident_wrong": 0, "macro_f1": 0.8568514794199706, "confusion_matrix": { "business": { "business": 24, "science and technology": 0, "sports": 0, "world": 0, "__failed__": 1 }, "science and technology": { "business": 7, "science and technology": 13, "sports": 1, "world": 4, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 24, "__failed__": 1 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 93, "accuracy": 0.93, "wilson95": [ 0.8625048526092651, 0.9656807389327273 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 93, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.93, "brier": 0.10989785440254173, "nll": 0.23682853819063854, "ece10": 0.04231813907940636, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.9157407407407407, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 2, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 2, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 2, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 2, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 2, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 2, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 2, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 1, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 1, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 32, "accuracy": 0.32, "wilson95": [ 0.23669147324999093, 0.41662618610452384 ], "valid": 55, "failures": 45, "coverage": 0.55, "heads": 100, "head_correct": 32, "native_tie_decisions_changed": 0, "abstained_records": 45, "truncated_records": 0, "head_accuracy": 0.32, "brier": 0.6403488413816059, "nll": 1.4253359730586022, "ece10": 0.2663930895363361, "calibration_n": 55, "confident_wrong": 6, "macro_f1": 0.37287581699346406, "confusion_matrix": { "anger": { "anger": 5, "fear": 1, "joy": 0, "love": 1, "sadness": 2, "surprise": 0, "__failed__": 8 }, "fear": { "anger": 1, "fear": 10, "joy": 0, "love": 0, "sadness": 1, "surprise": 0, "__failed__": 5 }, "joy": { "anger": 0, "fear": 0, "joy": 2, "love": 1, "sadness": 3, "surprise": 2, "__failed__": 9 }, "love": { "anger": 1, "fear": 0, "joy": 0, "love": 7, "sadness": 2, "surprise": 0, "__failed__": 7 }, "sadness": { "anger": 0, "fear": 2, "joy": 0, "love": 1, "sadness": 8, "surprise": 0, "__failed__": 5 }, "surprise": { "anger": 1, "fear": 0, "joy": 1, "love": 1, "sadness": 2, "surprise": 0, "__failed__": 11 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2524, "accuracy": 0.8561736770691994, "wilson95": [ 0.8430426106601469, 0.8683777109580008 ], "valid": 2794, "failures": 154, "coverage": 0.9477611940298507, "heads": 2948, "head_correct": 2524, "native_tie_decisions_changed": 0, "abstained_records": 154, "truncated_records": 0, "head_accuracy": 0.8561736770691994, "brier": 0.15051512438194076, "nll": 0.35455324924856974, "ece10": 0.009740221551587318, "calibration_n": 2794, "confident_wrong": 56, "macro_f1": 0.8729293605942599, "confusion_matrix": { "alarm": { "alarm": 90, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 52, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 2 }, "calendar": { "alarm": 1, "audio": 0, "calendar": 370, "cooking": 0, "datetime": 5, "email": 0, "general": 0, "iot": 0, "lists": 2, "music": 0, "news": 4, "play": 0, "qa": 4, "recommendation": 6, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 6 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 66, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 2 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 96, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 1 }, "email": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 227, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 4, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 32 }, "general": { "alarm": 0, "audio": 0, "calendar": 9, "cooking": 6, "datetime": 0, "email": 0, "general": 80, "iot": 3, "lists": 0, "music": 1, "news": 3, "play": 4, "qa": 36, "recommendation": 7, "social": 1, "takeaway": 0, "transport": 2, "weather": 3, "__failed__": 32 }, "iot": { "alarm": 0, "audio": 2, "calendar": 0, "cooking": 1, "datetime": 0, "email": 0, "general": 1, "iot": 204, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 1, "__failed__": 3 }, "lists": { "alarm": 0, "audio": 1, "calendar": 7, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 1, "lists": 122, "music": 2, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 6 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 1, "music": 69, "news": 0, "play": 2, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 2 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 113, "play": 0, "qa": 5, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 4 }, "play": { "alarm": 0, "audio": 3, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 3, "iot": 2, "lists": 0, "music": 31, "news": 1, "play": 336, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 7 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 10, "iot": 0, "lists": 0, "music": 1, "news": 5, "play": 0, "qa": 231, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 1, "weather": 2, "__failed__": 35 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 2, "play": 1, "qa": 5, "recommendation": 71, "social": 0, "takeaway": 2, "transport": 0, "weather": 1, "__failed__": 9 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 8, "iot": 0, "lists": 0, "music": 0, "news": 6, "play": 0, "qa": 0, "recommendation": 0, "social": 83, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 8 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 51, "transport": 0, "weather": 0, "__failed__": 1 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 117, "weather": 0, "__failed__": 2 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 146, "__failed__": 2 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2424, "accuracy": 0.8222523744911805, "wilson95": [ 0.8080353011224061, 0.835630704151156 ], "valid": 2813, "failures": 135, "coverage": 0.9542062415196744, "heads": 2948, "head_correct": 2424, "native_tie_decisions_changed": 0, "abstained_records": 135, "truncated_records": 0, "head_accuracy": 0.8222523744911805, "brier": 0.21117028285992628, "nll": 0.5046371607872124, "ece10": 0.015954658234518262, "calibration_n": 2813, "confident_wrong": 44, "macro_f1": 0.8303859827372283, "confusion_matrix": { "alarm": { "alarm": 90, "audio": 2, "calendar": 4, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 42, "calendar": 1, "cooking": 0, "datetime": 2, "email": 0, "general": 8, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 1 }, "calendar": { "alarm": 3, "audio": 0, "calendar": 360, "cooking": 0, "datetime": 4, "email": 2, "general": 2, "iot": 2, "lists": 2, "music": 1, "news": 3, "play": 2, "qa": 6, "recommendation": 2, "social": 0, "takeaway": 1, "transport": 1, "weather": 2, "__failed__": 7 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 66, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 1 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 98, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 1 }, "email": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 226, "general": 4, "iot": 1, "lists": 0, "music": 0, "news": 2, "play": 1, "qa": 7, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 1, "weather": 1, "__failed__": 22 }, "general": { "alarm": 0, "audio": 2, "calendar": 14, "cooking": 7, "datetime": 0, "email": 1, "general": 74, "iot": 3, "lists": 0, "music": 3, "news": 6, "play": 4, "qa": 34, "recommendation": 6, "social": 0, "takeaway": 0, "transport": 3, "weather": 4, "__failed__": 26 }, "iot": { "alarm": 2, "audio": 6, "calendar": 0, "cooking": 1, "datetime": 1, "email": 0, "general": 1, "iot": 192, "lists": 4, "music": 0, "news": 0, "play": 3, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 2 }, "lists": { "alarm": 1, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 3, "lists": 120, "music": 1, "news": 1, "play": 3, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 5 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 69, "news": 1, "play": 4, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "news": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 1, "news": 101, "play": 0, "qa": 7, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 7 }, "play": { "alarm": 0, "audio": 13, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 2, "iot": 1, "lists": 1, "music": 36, "news": 0, "play": 313, "qa": 4, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 11 }, "qa": { "alarm": 0, "audio": 2, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 8, "iot": 2, "lists": 0, "music": 2, "news": 7, "play": 0, "qa": 221, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 5, "__failed__": 37 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 2, "qa": 7, "recommendation": 73, "social": 0, "takeaway": 1, "transport": 4, "weather": 1, "__failed__": 2 }, "social": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 2, "general": 8, "iot": 0, "lists": 0, "music": 0, "news": 4, "play": 0, "qa": 3, "recommendation": 0, "social": 77, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 10 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 2, "datetime": 0, "email": 0, "general": 0, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 48, "transport": 0, "weather": 0, "__failed__": 1 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 2, "email": 0, "general": 1, "iot": 1, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 112, "weather": 1, "__failed__": 1 }, "weather": { "alarm": 1, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 2, "iot": 1, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 142, "__failed__": 1 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 819, "accuracy": 0.6982097186700768, "wilson95": [ 0.6713285823681161, 0.7237968582889234 ], "valid": 1150, "failures": 23, "coverage": 0.9803921568627451, "heads": 1173, "head_correct": 819, "native_tie_decisions_changed": 0, "abstained_records": 23, "truncated_records": 0, "head_accuracy": 0.6982097186700768, "cluster_bootstrap95": [ 0.6672839838782679, 0.7286962215805101 ], "independent_source_groups": 47, "brier": 0.4020210690250534, "nll": 0.7045983813594031, "ece10": 0.038764681651624805, "calibration_n": 1150, "confident_wrong": 3, "macro_f1": 0.6516257858884076, "confusion_matrix": { "negative": { "negative": 78, "neutral": 95, "positive": 9, "__failed__": 0 }, "neutral": { "negative": 29, "neutral": 510, "positive": 28, "__failed__": 23 }, "positive": { "negative": 19, "neutral": 151, "positive": 231, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "intern-decision-4b": { "label": "Intern-Decision 4B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 176, "accuracy": 0.88, "wilson95": [ 0.8276574790170326, 0.9180200729362449 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 176, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.1691601998402321, "nll": 0.3467404988045745, "ece10": 0.08280329905296234, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.8832706995841816, "confusion_matrix": { "world": { "world": 44, "sports": 0, "business": 2, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 45, "business": 2, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 42, "sci_tech": 7, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 5, "sci_tech": 45, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 96, "accuracy": 0.48, "wilson95": [ 0.411791663783233, 0.5489621492718576 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 96, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.6820512137603015, "nll": 1.3352312940040096, "ece10": 0.1828349969637072, "calibration_n": 200, "confident_wrong": 3, "macro_f1": 0.36486195334046134, "confusion_matrix": { "sadness": { "sadness": 41, "joy": 2, "love": 2, "anger": 10, "fear": 2, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 31, "joy": 31, "love": 11, "anger": 3, "fear": 3, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 6, "joy": 3, "love": 3, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 13, "joy": 1, "love": 0, "anger": 13, "fear": 1, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 8, "joy": 1, "love": 1, "anger": 1, "fear": 8, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 1, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 188, "accuracy": 0.94, "wilson95": [ 0.8980683070423372, 0.9653478057456681 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 188, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.94, "brier": 0.11622449443020277, "nll": 0.22680387773699603, "ece10": 0.09502871874600977, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.9389685688129387, "confusion_matrix": { "false": { "false": 107, "true": 2, "__failed__": 0 }, "true": { "false": 10, "true": 81, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 183, "accuracy": 0.915, "wilson95": [ 0.8681041292843443, 0.9462542498225244 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 183, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.915, "brier": 0.1298854742891154, "nll": 0.2289904309321108, "ece10": 0.03679790150472889, "calibration_n": 200, "confident_wrong": 3, "macro_f1": 0.9062818710548802, "confusion_matrix": { "true": { "true": 61, "false": 16, "__failed__": 0 }, "false": { "true": 1, "false": 122, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.20671165317256404, "nll": 0.3892957829115978, "ece10": 0.15631112986001358, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8533333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.2927010747387659, "nll": 0.5522557905800421, "ece10": 0.13246286062420476, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.76, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 1, "it": 1, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.005318355959993595, "nll": 0.05376876699495597, "ece10": 0.05198587553478919, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.008457774374971179, "nll": 0.06759218607900098, "ece10": 0.06462404895032413, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 92, "accuracy": 0.92, "wilson95": [ 0.8500189229905127, 0.9589065385156194 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 92, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.92, "brier": 0.11252898303347124, "nll": 0.25011118477446326, "ece10": 0.09700771473600948, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.9164432034853789, "confusion_matrix": { "label_000": { "label_000": 24, "label_001": 1, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 4, "label_001": 18, "label_002": 1, "label_003": 2, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 47, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47, "brier": 0.724935608066404, "nll": 1.4622440284709968, "ece10": 0.25111734839652816, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.4538709150154488, "confusion_matrix": { "label_000": { "label_000": 10, "label_001": 2, "label_002": 1, "label_003": 0, "label_004": 4, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 10, "label_002": 1, "label_003": 0, "label_004": 5, "label_005": 0, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 10, "label_003": 0, "label_004": 4, "label_005": 2, "__failed__": 0 }, "label_003": { "label_000": 1, "label_001": 0, "label_002": 3, "label_003": 5, "label_004": 8, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 2, "label_002": 2, "label_003": 1, "label_004": 11, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 0, "label_002": 6, "label_003": 0, "label_004": 8, "label_005": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "label_000": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_001": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.001141033384744314, "nll": 0.021662392770999257, "ece10": 0.02130819996221669, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 82, "accuracy": 0.7387387387387387, "wilson95": [ 0.6500078016644905, 0.8114980042026286 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 82, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7387387387387387, "brier": 0.3470200503221405, "nll": 0.6174748228370467, "ece10": 0.07055398072925285, "calibration_n": 111, "confident_wrong": 0 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 71, "accuracy": 0.9861111111111112, "wilson95": [ 0.9254339785252386, 0.9975440481170943 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 71, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9861111111111112, "cluster_bootstrap95": [ 0.9583333333333334, 1.0 ], "independent_source_groups": 36, "brier": 0.03376049858979906, "nll": 0.10747066103427753, "ece10": 0.09474857999735453, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.09064059855553534, "nll": 0.2106554282375339, "ece10": 0.1814443254873485, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.020266286247421498, "nll": 0.08237168467060892, "ece10": 0.07656763491609081, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.1757310963910514, "nll": 0.2938744274171122, "ece10": 0.1138722029422968, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8562091503267973, "confusion_matrix": { "false": { "false": 7, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03069100998145194, "nll": 0.08920242365687367, "ece10": 0.07748039539621632, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.11233635655049186, "nll": 0.2530188094346694, "ece10": 0.20157618903594254, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.0350314387368284, "nll": 0.09438035959550564, "ece10": 0.0795464515961253, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 1, "roads": 3, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.18883667767828424, "nll": 0.34884663688478923, "ece10": 0.15089486296514878, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8250000000000001, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 1, "security": 0, "style": 2, "refactor": 0, "nit": 1, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.12979665286691952, "nll": 0.28019459057583407, "ece10": 0.15601045870554275, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9111111111111111, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 2, "test": 0, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.04668580173092594, "nll": 0.1203330751855605, "ece10": 0.1037885917399568, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9403508771929825, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.17873104289067065, "nll": 0.3160336141942818, "ece10": 0.1263579116121577, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.1846435552467503, "nll": 0.34570878907788416, "ece10": 0.10800248998583761, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8377777777777778, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 1, "threat": 1, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 1, "none": 5, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.18220610084949113, "nll": 0.35946997763324035, "ece10": 0.1352548991605486, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8565656565656565, "confusion_matrix": { "news_report": { "news_report": 4, "opinion": 0, "press_release": 1, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 2, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 2, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.028345228607896265, "nll": 0.0703268974872217, "ece10": 0.05961243769186011, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.003118886170777418, "nll": 0.040498974385624956, "ece10": 0.03933900318469371, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.0905148917935042, "nll": 0.1954400336519012, "ece10": 0.16492064461957068, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9411764705882353, "confusion_matrix": { "false": { "false": 8, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.003724053191196633, "nll": 0.03747551172852978, "ece10": 0.03595294133072768, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.04736509893237863, "nll": 0.11481902314545668, "ece10": 0.093772366164767, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.0933675681245804, "nll": 0.19661086226386182, "ece10": 0.12075168813237441, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8711111111111112, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 4, "office_supplies": 0, "software": 0, "client_entertainment": 1, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.16728754911542484, "nll": 0.27881266062516025, "ece10": 0.10705594541205302, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8875, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.4313817234831605, "nll": 0.6560113945351177, "ece10": 0.14104815271700977, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7077922077922078, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 5, "3": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06827799845638928, "nll": 0.1834553136901633, "ece10": 0.15897034881087818, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.16318239526261274, "nll": 0.3505585885597568, "ece10": 0.23776866395942464, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 4, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 3, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 13, "accuracy": 0.6190476190476191, "wilson95": [ 0.4087865377511576, 0.7924898863143551 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6190476190476191, "brier": 0.5093410496078561, "nll": 0.9429463078180501, "ece10": 0.12646643075964914, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.6297619047619047, "confusion_matrix": { "spelling": { "spelling": 2, "subject_verb_agreement": 0, "word_choice": 2, "punctuation": 0, "none": 0, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 3, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 0, "none": 3, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 2, "none": 3, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.013465249250885627, "nll": 0.07041423287186503, "ece10": 0.0667013643582118, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00979886977508077, "nll": 0.07098607085865595, "ece10": 0.06743975728269974, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 12, "accuracy": 0.8571428571428571, "wilson95": [ 0.6005862050302965, 0.9599060787285351 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.25130576608800415, "nll": 0.4810882855097451, "ece10": 0.07523392784089016, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.75, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.21979946326608596, "nll": 0.3793778016453882, "ece10": 0.10190035257803463, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8875, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.2283899136420164, "nll": 0.41732251118577646, "ece10": 0.12880535639530782, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7430555555555555, "confusion_matrix": { "0": { "0": 2, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.3222592056228707, "nll": 0.47769039535230534, "ece10": 0.17511561454318839, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6640316205533596, "confusion_matrix": { "false": { "false": 3, "true": 5, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.053502612447449915, "nll": 0.15924424789284433, "ece10": 0.1373880633314126, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06672408188198063, "nll": 0.1972923381904786, "ece10": 0.17057741984282132, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.07503475039677407, "nll": 0.1860766827675726, "ece10": 0.1627121211244264, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.07085330475651787, "nll": 0.1525112929535228, "ece10": 0.12414708823885755, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9372549019607843, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.17429715816025876, "nll": 0.3296411181426368, "ece10": 0.1698781280392193, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9393939393939394, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.09773246608374545, "nll": 0.2502161330575217, "ece10": 0.0956169689106758, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571427, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.3451556431244857, "nll": 0.5089160916565303, "ece10": 0.12958159437970942, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7017543859649122, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 4, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.08694754477157098, "nll": 0.19111412229831204, "ece10": 0.1315810558017505, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.4055185955468484, "nll": 0.9519249917354554, "ece10": 0.15417911232990952, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7814285714285714, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.2705250327256135, "nll": 0.40914275406064227, "ece10": 0.19712580371986427, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.746031746031746, "confusion_matrix": { "false": { "false": 7, "true": 3, "__failed__": 0 }, "true": { "false": 1, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 12, "accuracy": 0.8, "wilson95": [ 0.5481455128483065, 0.9295245065301844 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.331900272093372, "nll": 0.4949660854620926, "ece10": 0.23459183992554666, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7963800904977376, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.05939542784087421, "nll": 0.155346234690056, "ece10": 0.1287757223401566, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 1, "tech": 0, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.07649274015630536, "nll": 0.19080525372641446, "ece10": 0.16742258452476863, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 16, "accuracy": 0.8421052631578947, "wilson95": [ 0.6243452472967093, 0.9447952836146585 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8421052631578947, "brier": 0.25960587147590836, "nll": 0.45185260098785696, "ece10": 0.1671785855389898, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8127705627705627, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.2142155753075587, "nll": 0.36476157023118905, "ece10": 0.12796507268291016, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8712121212121212, "confusion_matrix": { "false": { "false": 10, "true": 1, "__failed__": 0 }, "true": { "false": 1, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.013281556586531285, "nll": 0.06976998563400884, "ece10": 0.06606434133164538, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 11, "accuracy": 0.55, "wilson95": [ 0.34208534245034233, 0.7418021417443759 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.5085614925143638, "nll": 0.9368658883632804, "ece10": 0.18348619739341646, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.5280303030303031, "confusion_matrix": { "0": { "0": 3, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 2, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 1, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 20, "accuracy": 1.0, "wilson95": [ 0.8388748419471806, 1.0 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.018459725746260806, "nll": 0.0713896642019033, "ece10": 0.06430929092748958, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 17, "accuracy": 0.85, "wilson95": [ 0.6395811352592431, 0.9476312541037835 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.24290284009629892, "nll": 0.4009167293079708, "ece10": 0.05140066871074337, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.849624060150376, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.139286265034887, "nll": 0.3198224212712166, "ece10": 0.16373611626473616, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9142857142857143, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 2, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 557, "accuracy": 0.6188888888888889, "wilson95": [ 0.5867178705857508, 0.6500493167473254 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 721, "native_tie_decisions_changed": 8, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6719478098788444, "cluster_bootstrap95": [ 0.5837645734672641, 0.6561690492891651 ], "independent_source_groups": 596, "brier": 0.43088094857091547, "nll": 0.7964280958498972, "ece10": 0.08454795727714084, "calibration_n": 1073, "confident_wrong": 24 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 426, "accuracy": 0.7421602787456446, "wilson95": [ 0.704847021312435, 0.7762537992610229 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 778, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8311965811965812, "brier": 0.25474885419098114, "nll": 0.4940515885865175, "ece10": 0.041241701964506466, "calibration_n": 936, "confident_wrong": 8 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 293, "accuracy": 0.4185714285714286, "wilson95": [ 0.3825676662294505, 0.45546404024176246 ], "valid": 689, "failures": 11, "coverage": 0.9842857142857143, "heads": 1088, "head_correct": 591, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5431985294117647, "cluster_bootstrap95": [ 0.3839541547277937, 0.4551952120093217 ], "independent_source_groups": 650, "brier": 0.5642385759723391, "nll": 1.0914044530845277, "ece10": 0.09458270879747598, "calibration_n": 1066, "confident_wrong": 16 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 155, "accuracy": 0.3875, "wilson95": [ 0.34104299567214824, 0.43609727055306846 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1611, "native_tie_decisions_changed": 4, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8055, "brier": 0.27596396584539046, "nll": 0.4825194695843495, "ece10": 0.07167128919200857, "calibration_n": 2000, "confident_wrong": 4 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.022070639009556945, "nll": 0.08425553559631486, "ece10": 0.07616540674752034, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.032639278284052574, "nll": 0.09520232071604141, "ece10": 0.08205843006905261, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06502922688693115, "nll": 0.16469413995305912, "ece10": 0.1348470446641564, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.08793634613110765, "nll": 0.19870153932997536, "ece10": 0.10578199698811021, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 1, "purchase": 0, "inspection": 0, "design": 6, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 39, "accuracy": 0.975, "wilson95": [ 0.8711863103652591, 0.9955731684973186 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.975, "brier": 0.07962531462520955, "nll": 0.21098026213930793, "ece10": 0.14553161341937404, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9749019607843138, "confusion_matrix": { "review": { "review": 7, "it": 0, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 39, "accuracy": 0.975, "wilson95": [ 0.8711863103652591, 0.9955731684973186 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.975, "brier": 0.05392335546524365, "nll": 0.16235843267220149, "ece10": 0.13467875857486386, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9749019607843138, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 1, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 37, "accuracy": 0.925, "wilson95": [ 0.8013576647568946, 0.9741639742254119 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.1576769716099285, "nll": 0.35156690462730505, "ece10": 0.14399141344575428, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9198592257415786, "confusion_matrix": { "review": { "review": 5, "it": 0, "documents": 2, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 39, "accuracy": 0.975, "wilson95": [ 0.8711863103652591, 0.9955731684973186 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.975, "brier": 0.09726124946643003, "nll": 0.2496548521915985, "ece10": 0.13873701630174107, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9749019607843138, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 215, "accuracy": 0.5375, "wilson95": [ 0.4885138311983594, 0.5857727467265683 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 215, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5375, "brier": 0.5978068375472403, "nll": 1.202971106180811, "ece10": 0.13629370461514548, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.46588626466945054, "confusion_matrix": { "sadness": { "sadness": 99, "joy": 3, "love": 6, "anger": 6, "fear": 5, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 51, "joy": 48, "love": 16, "anger": 2, "fear": 0, "surprise": 1, "__failed__": 0 }, "love": { "sadness": 16, "joy": 5, "love": 10, "anger": 4, "fear": 2, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 27, "joy": 0, "love": 4, "anger": 31, "fear": 0, "surprise": 2, "__failed__": 0 }, "fear": { "sadness": 19, "joy": 1, "love": 1, "anger": 5, "fear": 25, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 5, "joy": 1, "love": 1, "anger": 1, "fear": 0, "surprise": 2, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 273, "accuracy": 0.91, "wilson95": [ 0.8722229892013517, 0.9374097743945206 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 273, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.91, "brier": 0.14105907197056708, "nll": 0.3841487735586371, "ece10": 0.11819531048887531, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 332, "accuracy": 0.83, "wilson95": [ 0.7900908679413299, 0.8636310177980344 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 332, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.83, "brier": 0.24673461057259347, "nll": 0.3796577059269472, "ece10": 0.07382688932282526, "calibration_n": 400, "confident_wrong": 8, "macro_f1": 0.7946301833227629, "confusion_matrix": { "false": { "false": 249, "true": 60, "__failed__": 0 }, "true": { "false": 8, "true": 83, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 355, "accuracy": 0.8875, "wilson95": [ 0.8527764306768796, 0.9148515412140404 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 355, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8875, "brier": 0.17735429526164648, "nll": 0.35333220862893006, "ece10": 0.05885835240472825, "calibration_n": 400, "confident_wrong": 5, "macro_f1": 0.8767164558804557, "confusion_matrix": { "world": { "world": 88, "sports": 6, "business": 8, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 2, "sports": 121, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 3, "sports": 0, "business": 61, "sci_tech": 8, "__failed__": 0 }, "sci_tech": { "world": 3, "sports": 0, "business": 14, "sci_tech": 85, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 368, "accuracy": 0.92, "wilson95": [ 0.8892454231693245, 0.9427642495898665 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 368, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.92, "brier": 0.14275840531709202, "nll": 0.2463970440779915, "ece10": 0.056226157439598856, "calibration_n": 400, "confident_wrong": 5, "macro_f1": 0.9117947021692991, "confusion_matrix": { "true": { "true": 123, "false": 30, "__failed__": 0 }, "false": { "true": 2, "false": 245, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 234, "accuracy": 0.78, "wilson95": [ 0.7297473820840871, 0.8231725540301672 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 234, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.78, "brier": 0.3294719490910712, "nll": 0.8225339555415856, "ece10": 0.11300810651352103, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 378, "accuracy": 0.945, "wilson95": [ 0.9181325521920267, 0.9634015058504493 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 378, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.945, "brier": 0.09945347393173762, "nll": 0.2036477066585732, "ece10": 0.10321183885603877, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.9449876222149984, "confusion_matrix": { "false": { "false": 186, "true": 2, "__failed__": 0 }, "true": { "false": 20, "true": 192, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 309, "accuracy": 0.7725, "wilson95": [ 0.7289390669951599, 0.8108767325926485 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 309, "native_tie_decisions_changed": 2, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7725, "brier": 0.3014412483022966, "nll": 0.45213769855746305, "ece10": 0.06391287245027952, "calibration_n": 400, "confident_wrong": 4, "macro_f1": 0.7650991552604238, "confusion_matrix": { "false": { "false": 190, "true": 10, "__failed__": 0 }, "true": { "false": 81, "true": 119, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 74, "accuracy": 0.74, "wilson95": [ 0.6462901055081282, 0.8159530153525186 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 74, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.74, "brier": 0.32313762733578655, "nll": 0.4943355399679569, "ece10": 0.06735150686443951, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.7267759562841529, "confusion_matrix": { "yes": { "yes": 48, "no": 6, "__failed__": 0 }, "no": { "yes": 20, "no": 26, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 42, "accuracy": 0.42, "wilson95": [ 0.3279838267435473, 0.5179351329695703 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 42, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42, "brier": 0.7917682028593841, "nll": 1.9465548801330046, "ece10": 0.24280038985638042, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.45256143162393164, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 2, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 3, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 7, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 1, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 2, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "statement": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 2, "statement": 2, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 4, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 3, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 7, "loan_payment": 1, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 2, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 4, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 3, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 3, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 3, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 1, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 50, "accuracy": 0.5, "wilson95": [ 0.4038315303659956, 0.5961684696340044 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 146, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.73, "brier": 0.3880791127020676, "nll": 0.663193751200743, "ece10": 0.1435123715464945, "calibration_n": 200, "confident_wrong": 2 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 38, "accuracy": 0.38, "wilson95": [ 0.29097599252478734, 0.47790244704488916 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 120, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5631584355860778, "nll": 1.1392558541029292, "ece10": 0.17142085451073608, "calibration_n": 200, "confident_wrong": 12 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 74, "accuracy": 0.74, "wilson95": [ 0.6462901055081282, 0.8159530153525186 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 74, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.74, "brier": 0.3582309675317054, "nll": 0.8885298902292523, "ece10": 0.07097551487558473, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.7136751517583161, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 13, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 5, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 10, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 7, "syllabus": 0, "news_article": 2, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 2, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 3, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 7, "accuracy": 0.07, "wilson95": [ 0.03431926106727268, 0.137495147390735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 213, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5325, "brier": 0.6762081613767137, "nll": 1.3574977187347657, "ece10": 0.23166758326550485, "calibration_n": 400, "confident_wrong": 36 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 61, "accuracy": 0.61, "wilson95": [ 0.5120301704584073, 0.6998312599360558 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 61, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.5339455742794026, "nll": 1.2430751386548746, "ece10": 0.12316670614484883, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6148765352294764, "confusion_matrix": { "politics": { "politics": 5, "business": 0, "technology": 0, "science": 3, "health": 3, "sports": 0, "entertainment": 1, "world": 1, "culture": 0, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 1, "technology": 8, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 2, "business": 1, "technology": 1, "science": 6, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 6, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 2, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 6, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "world": { "politics": 2, "business": 2, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 3, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 3, "business": 0, "technology": 0, "science": 1, "health": 1, "sports": 0, "entertainment": 2, "world": 0, "culture": 3, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 2, "culture": 0, "weather": 5, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 38, "accuracy": 0.38, "wilson95": [ 0.29097599252478734, 0.47790244704488916 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.38, "brier": 0.7918861903528084, "nll": 2.0399943162990044, "ece10": 0.15508218818391142, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.33603702350505416, "confusion_matrix": { "biology": { "biology": 6, "medicine": 0, "chemistry": 1, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 9, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 4, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 2, "medicine": 3, "chemistry": 1, "physics": 5, "computer_science": 0, "mathematics": 0, "earth_science": 3, "engineering": 1, "economics": 1, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 2, "chemistry": 2, "physics": 0, "computer_science": 2, "mathematics": 0, "earth_science": 4, "engineering": 4, "economics": 1, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 1, "chemistry": 1, "physics": 1, "computer_science": 1, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 2, "medicine": 3, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 5, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 1, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 1, "economics": 3, "other": 0, "__failed__": 0 }, "other": { "biology": 2, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 1, "mathematics": 0, "earth_science": 4, "engineering": 3, "economics": 1, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 11, "accuracy": 0.11, "wilson95": [ 0.06254196357064568, 0.18631296503080313 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 74, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.37, "brier": 0.6261783617918286, "nll": 1.269884478842711, "ece10": 0.22883455366477456, "calibration_n": 100, "confident_wrong": 13 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 39, "accuracy": 0.39, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 134, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.67, "brier": 0.2460221515479381, "nll": 0.4616273800507924, "ece10": 0.09705475924549523, "calibration_n": 100, "confident_wrong": 3 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 87, "accuracy": 0.87, "wilson95": [ 0.7901964855992357, 0.9224283257275948 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 87, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.21012251267846135, "nll": 0.418390058749775, "ece10": 0.051370162045092066, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.8708925090504037, "confusion_matrix": { "negative": { "negative": 35, "neutral": 1, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 7, "neutral": 26, "positive": 0, "__failed__": 0 }, "positive": { "negative": 2, "neutral": 3, "positive": 26, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 19, "accuracy": 0.19, "wilson95": [ 0.12514751509768735, 0.27778845379064376 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 106, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.53, "brier": 0.2585411849718488, "nll": 0.407382101750857, "ece10": 0.08325237205752889, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 239, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7966666666666666, "brier": 0.2901479826675928, "nll": 0.5828785050111559, "ece10": 0.032227021975012066, "calibration_n": 300, "confident_wrong": 6 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 54, "accuracy": 0.54, "wilson95": [ 0.44264860323368227, 0.6343919169097589 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 54, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.54, "brier": 0.6055143623112551, "nll": 1.7535693084033193, "ece10": 0.15320033413877912, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.4766455766455766, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 1, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 4, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 3, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 3, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 2, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 7, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 2, "tax_document": 3, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 4, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 35, "accuracy": 0.35, "wilson95": [ 0.26364248243097527, 0.44745556703112027 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35, "brier": 0.8571585140002503, "nll": 2.0851052434719595, "ece10": 0.2531939232059975, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.33705526726360063, "confusion_matrix": { "billing": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 3, "account_access": 4, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 4, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 1, "onboarding": 0, "integrations": 4, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 2, "integrations": 4, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 7, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 1, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 2, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 2, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "other": { "billing": 2, "account_access": 0, "shipping": 4, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 1, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 130, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43333333333333335, "brier": 0.7021363826317333, "nll": 1.5990930134245052, "ece10": 0.21084675382389628, "calibration_n": 300, "confident_wrong": 2 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.6198589286015906, "nll": 1.4422625373380564, "ece10": 0.17525570840898058, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.5498829873829875, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 5, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 1, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 2, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 2, "loyalty": 2, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 4, "car": 0, "visa_docs": 1, "loyalty": 3, "disruption": 1, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 5, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 4, "car": 2, "visa_docs": 1, "loyalty": 1, "disruption": 1, "other": 1, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 94, "accuracy": 0.94, "wilson95": [ 0.8752318455410406, 0.9722138760368118 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 94, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.94, "brier": 0.10195349287464665, "nll": 0.2464381166882715, "ece10": 0.12559483452888984, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.9397919167667067, "confusion_matrix": { "business": { "business": 23, "science and technology": 2, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 1, "science and technology": 22, "sports": 1, "world": 1, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 1, "sports": 0, "world": 24, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "refund not showing up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "activating a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.67673270919035, "nll": 1.4185048559193012, "ece10": 0.16505799287232795, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.47036975142965926, "confusion_matrix": { "anger": { "anger": 9, "fear": 2, "joy": 1, "love": 0, "sadness": 5, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 1, "fear": 13, "joy": 1, "love": 0, "sadness": 2, "surprise": 0, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 10, "love": 0, "sadness": 4, "surprise": 2, "__failed__": 0 }, "love": { "anger": 1, "fear": 0, "joy": 2, "love": 6, "sadness": 8, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 1, "fear": 3, "joy": 2, "love": 2, "sadness": 8, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 1, "joy": 6, "love": 0, "sadness": 6, "surprise": 2, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2448, "accuracy": 0.830393487109905, "wilson95": [ 0.8164183736355343, 0.8435086675607283 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2448, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.830393487109905, "brier": 0.26364656316848145, "nll": 0.6438500464827202, "ece10": 0.11012517901068533, "calibration_n": 2948, "confident_wrong": 4, "macro_f1": 0.8309611352920196, "confusion_matrix": { "alarm": { "alarm": 90, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 1, "email": 0, "general": 1, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 2, "audio": 49, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 3, "audio": 0, "calendar": 348, "cooking": 1, "datetime": 8, "email": 0, "general": 14, "iot": 0, "lists": 3, "music": 0, "news": 3, "play": 0, "qa": 5, "recommendation": 13, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 68, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 87, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 11, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 229, "general": 33, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 7, "cooking": 5, "datetime": 0, "email": 0, "general": 124, "iot": 2, "lists": 0, "music": 1, "news": 1, "play": 4, "qa": 22, "recommendation": 14, "social": 1, "takeaway": 0, "transport": 3, "weather": 3, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 3, "calendar": 0, "cooking": 11, "datetime": 0, "email": 0, "general": 11, "iot": 185, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 3, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 11, "iot": 1, "lists": 123, "music": 3, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 9, "iot": 0, "lists": 0, "music": 67, "news": 0, "play": 2, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 10, "iot": 0, "lists": 0, "music": 0, "news": 99, "play": 1, "qa": 13, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 14, "calendar": 0, "cooking": 0, "datetime": 4, "email": 0, "general": 20, "iot": 1, "lists": 1, "music": 63, "news": 1, "play": 278, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 30, "iot": 0, "lists": 0, "music": 1, "news": 2, "play": 0, "qa": 246, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 2, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 3, "play": 1, "qa": 3, "recommendation": 80, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 3, "general": 25, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 75, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 2, "datetime": 1, "email": 0, "general": 2, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 2, "social": 0, "takeaway": 48, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 7, "social": 0, "takeaway": 0, "transport": 111, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 8, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 141, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2126, "accuracy": 0.7211668928086838, "wilson95": [ 0.7046997474720419, 0.7370583950946553 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2126, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7211668928086838, "brier": 0.3845196564692118, "nll": 0.9669432309805042, "ece10": 0.10313580175778042, "calibration_n": 2948, "confident_wrong": 2, "macro_f1": 0.7303267063776697, "confusion_matrix": { "alarm": { "alarm": 86, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 2, "email": 0, "general": 1, "iot": 1, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 3, "audio": 39, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 15, "iot": 2, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 10, "audio": 0, "calendar": 313, "cooking": 1, "datetime": 8, "email": 3, "general": 33, "iot": 2, "lists": 3, "music": 0, "news": 2, "play": 1, "qa": 5, "recommendation": 15, "social": 1, "takeaway": 2, "transport": 1, "weather": 0, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 64, "datetime": 0, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 88, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 10, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 1, "datetime": 2, "email": 220, "general": 34, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 3, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 10, "cooking": 5, "datetime": 1, "email": 1, "general": 121, "iot": 2, "lists": 0, "music": 1, "news": 1, "play": 2, "qa": 21, "recommendation": 13, "social": 0, "takeaway": 0, "transport": 4, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 3, "audio": 9, "calendar": 1, "cooking": 8, "datetime": 1, "email": 0, "general": 16, "iot": 168, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 6, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 0, "email": 0, "general": 15, "iot": 2, "lists": 118, "music": 2, "news": 0, "play": 1, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 2, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 12, "iot": 0, "lists": 2, "music": 58, "news": 1, "play": 2, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 1, "general": 23, "iot": 0, "lists": 1, "music": 0, "news": 73, "play": 0, "qa": 20, "recommendation": 3, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "play": { "alarm": 3, "audio": 28, "calendar": 2, "cooking": 0, "datetime": 3, "email": 0, "general": 73, "iot": 6, "lists": 8, "music": 110, "news": 1, "play": 142, "qa": 3, "recommendation": 4, "social": 0, "takeaway": 1, "transport": 1, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 43, "iot": 2, "lists": 1, "music": 1, "news": 2, "play": 0, "qa": 230, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 1, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 2, "qa": 4, "recommendation": 78, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 2, "general": 32, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 62, "takeaway": 4, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 5, "datetime": 2, "email": 0, "general": 8, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 38, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 1, "email": 0, "general": 6, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 8, "social": 0, "takeaway": 0, "transport": 102, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 4, "email": 0, "general": 14, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 126, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 802, "accuracy": 0.6837169650468883, "wilson95": [ 0.6565421169630886, 0.7096924312588125 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 802, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6837169650468883, "cluster_bootstrap95": [ 0.6501305291224586, 0.7172548670386545 ], "independent_source_groups": 47, "brier": 0.4376028188402158, "nll": 0.7619601692986305, "ece10": 0.03399988765129784, "calibration_n": 1173, "confident_wrong": 28, "macro_f1": 0.6563645375526564, "confusion_matrix": { "negative": { "negative": 120, "neutral": 48, "positive": 14, "__failed__": 0 }, "neutral": { "negative": 103, "neutral": 402, "positive": 85, "__failed__": 0 }, "positive": { "negative": 50, "neutral": 71, "positive": 280, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "jev-omni-12b-nf4": { "label": "Jev-Omni 12B NF4 (local variant)", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 170, "accuracy": 0.85, "wilson95": [ 0.7939442071583334, 0.89286406437758 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 170, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.25455481260504553, "nll": 0.5953611813787599, "ece10": 0.11050431711854694, "calibration_n": 200, "confident_wrong": 15, "macro_f1": 0.8525703098071519, "confusion_matrix": { "world": { "world": 43, "sports": 0, "business": 2, "sci_tech": 5, "__failed__": 0 }, "sports": { "world": 3, "sports": 44, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 1, "sports": 0, "business": 47, "sci_tech": 3, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 14, "sci_tech": 36, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 106, "accuracy": 0.53, "wilson95": [ 0.4609168291500183, 0.5979524512673458 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 106, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.53, "brier": 0.7651785711540028, "nll": 2.1539281860830037, "ece10": 0.32681595731981217, "calibration_n": 200, "confident_wrong": 45, "macro_f1": 0.4156661532307924, "confusion_matrix": { "sadness": { "sadness": 43, "joy": 1, "love": 2, "anger": 3, "fear": 8, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 17, "joy": 33, "love": 7, "anger": 3, "fear": 18, "surprise": 1, "__failed__": 0 }, "love": { "sadness": 2, "joy": 4, "love": 5, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 7, "joy": 2, "love": 0, "anger": 14, "fear": 4, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 5, "joy": 2, "love": 0, "anger": 1, "fear": 11, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 193, "accuracy": 0.965, "wilson95": [ 0.9295293884777092, 0.9829444579914328 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 193, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.965, "brier": 0.049948476686986425, "nll": 0.08274451575821644, "ece10": 0.02284429157106109, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.964681248265597, "confusion_matrix": { "false": { "false": 106, "true": 3, "__failed__": 0 }, "true": { "false": 4, "true": 87, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 176, "accuracy": 0.88, "wilson95": [ 0.8276574790170326, 0.9180200729362449 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 176, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.17713571888573232, "nll": 0.2836447403326122, "ece10": 0.05995903116432265, "calibration_n": 200, "confident_wrong": 8, "macro_f1": 0.8697916666666667, "confusion_matrix": { "true": { "true": 60, "false": 17, "__failed__": 0 }, "false": { "true": 7, "false": 116, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.1465534667993442, "nll": 0.257865345906627, "ece10": 0.11453470522047672, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 0, "review": 3, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.10408840141722603, "nll": 0.20407438778260326, "ece10": 0.14321385710870738, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.12111318248050428, "nll": 0.45291132628193426, "ece10": 0.061562465374049906, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9142857142857144, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 1, "work": 2, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0004610041498517847, "nll": 0.011451888161454161, "ece10": 0.011303944102668542, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 85, "accuracy": 0.85, "wilson95": [ 0.7671644040916763, 0.9069401471634337 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 85, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.2786602046523779, "nll": 0.7507602014838137, "ece10": 0.130572742538773, "calibration_n": 100, "confident_wrong": 10, "macro_f1": 0.8287222714634879, "confusion_matrix": { "label_000": { "label_000": 25, "label_001": 0, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 11, "label_001": 10, "label_002": 1, "label_003": 3, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 46, "accuracy": 0.46, "wilson95": [ 0.36560808309024106, 0.5573513967663177 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 46, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.46, "brier": 0.9080267525397255, "nll": 2.584058554783889, "ece10": 0.41042016492434796, "calibration_n": 100, "confident_wrong": 30, "macro_f1": 0.4565447097705162, "confusion_matrix": { "label_000": { "label_000": 8, "label_001": 3, "label_002": 0, "label_003": 0, "label_004": 6, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 8, "label_002": 0, "label_003": 0, "label_004": 7, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 8, "label_003": 2, "label_004": 4, "label_005": 2, "__failed__": 0 }, "label_003": { "label_000": 1, "label_001": 0, "label_002": 1, "label_003": 7, "label_004": 8, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 1, "label_003": 1, "label_004": 13, "label_005": 1, "__failed__": 0 }, "label_005": { "label_000": 2, "label_001": 0, "label_002": 3, "label_003": 1, "label_004": 8, "label_005": 2, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 52, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.6251886551715288, "nll": 2.6726959140532314, "ece10": 0.16504980293469285, "calibration_n": 100, "confident_wrong": 10, "macro_f1": 0.35551146384479726, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 2, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 1, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 1, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 2, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 1, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 1, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 1, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 1, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 1, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 1, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 1, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 1, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 1, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 1, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 1, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 1, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 1, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 1, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 1, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 1, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 1, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 1, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.008447633401916056, "nll": 0.017292446540635015, "ece10": 0.013356560662270634, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 81, "accuracy": 0.7297297297297297, "wilson95": [ 0.6404595664697532, 0.8036309259684178 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 81, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7297297297297297, "brier": 0.36241011754052893, "nll": 0.7009520049975371, "ece10": 0.06891966803459705, "calibration_n": 111, "confident_wrong": 4 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 69, "accuracy": 0.9583333333333334, "wilson95": [ 0.8845072325868389, 0.9857291925330748 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9583333333333334, "cluster_bootstrap95": [ 0.9027777777777778, 1.0 ], "independent_source_groups": 36, "brier": 0.057716952360150606, "nll": 0.10514588774418072, "ece10": 0.04424160344171462, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.008057553742560552, "nll": 0.033788217347396635, "ece10": 0.03150394504744557, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.022931549492350427, "nll": 0.053407255662239306, "ece10": 0.045662698022759136, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 22, "accuracy": 1.0, "wilson95": [ 0.8513451253913791, 1.0 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02919288421274012, "nll": 0.06683861475184526, "ece10": 0.05732839173985648, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 10, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0005537612950355608, "nll": 0.009432179535840697, "ece10": 0.009237014398253474, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.12952358420225912, "nll": 0.28657362875111175, "ece10": 0.07551853499338743, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8761904761904761, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.06914841691300166, "nll": 0.10418355202838575, "ece10": 0.05651729331973622, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 1, "roads": 3, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.3088346500866399, "nll": 0.7157752445784576, "ece10": 0.15962176254977023, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.8444444444444444, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 1, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 12, "accuracy": 0.8, "wilson95": [ 0.5481455128483065, 0.9295245065301844 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.19086335662223622, "nll": 0.5159735867148239, "ece10": 0.10611098153613749, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7984126984126984, "confusion_matrix": { "feature": { "feature": 2, "fix": 0, "refactor": 1, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 2, "test": 0, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 2, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 1, "chore": 3, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.08831425507885948, "nll": 0.1328290511762235, "ece10": 0.0673847695306195, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9403508771929825, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.1631339565537146, "nll": 0.22857529869391385, "ece10": 0.12239047262066727, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.10245319786737227, "nll": 0.20083010240060697, "ece10": 0.09520567833067237, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9623931623931623, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 2, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 6, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.1417921800787839, "nll": 0.35390419075122154, "ece10": 0.06825062647454319, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9088023088023087, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 4, "press_release": 1, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 1, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 3, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0002585337338300517, "nll": 0.007443210771032251, "ece10": 0.007341834327198325, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0009111747695643739, "nll": 0.011077640381637013, "ece10": 0.010705184690391284, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.09045663958424455, "nll": 0.133130559128872, "ece10": 0.06281555021421517, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9411764705882353, "confusion_matrix": { "false": { "false": 8, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.11846568323344393, "nll": 0.3368888519925175, "ece10": 0.05431719312873736, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 2, "contract": 0, "resume": 1, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.016205237792044164, "nll": 0.04549537163263306, "ece10": 0.03786572273761782, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "meeting_request": { "meeting_request": 3, "question": 0, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.15214868859836236, "nll": 0.24016215595443471, "ece10": 0.10824057174531711, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8377777777777778, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 1, "software": 0, "client_entertainment": 1, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.15117326375745066, "nll": 0.282897440914055, "ece10": 0.09901919632848089, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.8875, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.10291313575674703, "nll": 0.3874745677019611, "ece10": 0.07931439503315094, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9415584415584416, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02288926510714136, "nll": 0.0773098240259819, "ece10": 0.0702127518053735, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.09574631637194933, "nll": 0.18756107340126243, "ece10": 0.046633725942645005, "calibration_n": 20, "confident_wrong": 1, "macro_f1": 0.9532467532467532, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 4, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 3, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 18, "accuracy": 0.8571428571428571, "wilson95": [ 0.6536393971937906, 0.9501898750027473 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.23936791909067734, "nll": 0.681731968335326, "ece10": 0.08641835254331631, "calibration_n": 21, "confident_wrong": 1, "macro_f1": 0.8632467532467534, "confusion_matrix": { "spelling": { "spelling": 3, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 0, "none": 0, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 3, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 4, "punctuation": 0, "none": 0, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 3, "none": 1, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00015734300566695554, "nll": 0.0068240465427898975, "ece10": 0.006784294864952717, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00023781855088764818, "nll": 0.007423338892309824, "ece10": 0.0073325718157295685, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 13, "accuracy": 0.9285714285714286, "wilson95": [ 0.6853129557584889, 0.9872777847521093 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.10314139646405442, "nll": 0.16725152266040957, "ece10": 0.03873199667615515, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.9047619047619048, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 1, "4": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.34807367423171964, "nll": 0.6668481139050081, "ece10": 0.161048694044035, "calibration_n": 16, "confident_wrong": 2, "macro_f1": 0.8166666666666667, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.46927830534361087, "nll": 0.7853871710814492, "ece10": 0.28163794540033016, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.6817460317460317, "confusion_matrix": { "0": { "0": 2, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.22046385623649684, "nll": 0.30985620130593405, "ece10": 0.15964770527228359, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8131868131868132, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00044341517294745183, "nll": 0.009874258454597098, "ece10": 0.009702734011715808, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.06705370241584803, "nll": 0.12219379090202791, "ece10": 0.053252538081043105, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9314285714285715, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 3, "data": 0, "model": 1, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.036759923407182295, "nll": 0.07106406285475556, "ece10": 0.05758172326047514, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9403508771929825, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.001556688738840195, "nll": 0.01731079885924562, "ece10": 0.016905877909415994, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.14792195465244934, "nll": 0.38844381070386313, "ece10": 0.0738403798768999, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.9393939393939394, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.21653212927604693, "nll": 0.7345611797492491, "ece10": 0.10989265746554003, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8396825396825397, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 3, "mexican": 1, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 2, "thai": 1, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 1, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.5751358084091941, "nll": 1.0514506219237703, "ece10": 0.30233801615337175, "calibration_n": 17, "confident_wrong": 3, "macro_f1": 0.6357142857142857, "confusion_matrix": { "false": { "false": 4, "true": 4, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.004883857375150155, "nll": 0.033809219723102274, "ece10": 0.03195910234262082, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 4, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.26935190028999484, "nll": 0.4618519419682641, "ece10": 0.1166422326797613, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.8261904761904763, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 1, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.22321441593573368, "nll": 0.39289421349881054, "ece10": 0.1232251673975211, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.8666666666666667, "confusion_matrix": { "false": { "false": 9, "true": 1, "__failed__": 0 }, "true": { "false": 1, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.15874845436444296, "nll": 0.2983905981463309, "ece10": 0.08546230586283816, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.9333333333333333, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.039685685100639426, "nll": 0.07775563147600575, "ece10": 0.05942825608968093, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 4, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.005392159560883422, "nll": 0.037254592148892055, "ece10": 0.03579879684432976, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 16, "accuracy": 0.8421052631578947, "wilson95": [ 0.6243452472967093, 0.9447952836146585 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8421052631578947, "brier": 0.27231831856749417, "nll": 0.5016744325432029, "ece10": 0.10297496460201165, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.6818181818181819, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.17854526300662185, "nll": 0.3304232358487231, "ece10": 0.12917826971756505, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.8583333333333334, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.004050484847964484, "nll": 0.015135391600404054, "ece10": 0.013975086764686912, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 17, "accuracy": 0.85, "wilson95": [ 0.6395811352592431, 0.9476312541037835 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.2635227085327675, "nll": 0.6324402273496279, "ece10": 0.13443469663430382, "calibration_n": 20, "confident_wrong": 2, "macro_f1": 0.7818181818181819, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 18, "accuracy": 0.9, "wilson95": [ 0.6989663547715128, 0.9721335187862319 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.14572779041850928, "nll": 0.4295651804761009, "ece10": 0.08508908603593796, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8825396825396826, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 2, "small_talk": 1, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 17, "accuracy": 0.85, "wilson95": [ 0.6395811352592431, 0.9476312541037835 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.1614035685478734, "nll": 0.23292684621323473, "ece10": 0.09454665753686267, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.849624060150376, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 21, "accuracy": 0.9545454545454546, "wilson95": [ 0.7820195416134267, 0.991930572378736 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9545454545454546, "brier": 0.061412136066063855, "nll": 0.12295581945241855, "ece10": 0.06721027430431678, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9545454545454546, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 5, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 573, "accuracy": 0.6366666666666667, "wilson95": [ 0.6047251567656826, 0.6674464697947693 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 739, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6887232059645852, "cluster_bootstrap95": [ 0.5948990182670704, 0.6743953013420767 ], "independent_source_groups": 596, "brier": 0.4885685205186063, "nll": 1.0404869091835542, "ece10": 0.17731652308481194, "calibration_n": 1073, "confident_wrong": 153 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 417, "accuracy": 0.7264808362369338, "wilson95": [ 0.6885986103910212, 0.761351797339551 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 773, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8258547008547008, "brier": 0.27173322021067753, "nll": 0.6585316487681464, "ece10": 0.07978328963116608, "calibration_n": 936, "confident_wrong": 61 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 366, "accuracy": 0.5228571428571429, "wilson95": [ 0.48583217792096, 0.5596326062274717 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 693, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6369485294117647, "cluster_bootstrap95": [ 0.4864081809841072, 0.5606303444635187 ], "independent_source_groups": 650, "brier": 0.5018306773163124, "nll": 1.016508996954666, "ece10": 0.1196453668335409, "calibration_n": 1088, "confident_wrong": 46 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 72, "accuracy": 0.18, "wilson95": [ 0.14545031532583302, 0.22063755304811672 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1301, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6505, "brier": 0.5066793844629077, "nll": 0.9556720622431415, "ece10": 0.16694098287680426, "calibration_n": 2000, "confident_wrong": 153 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.012181792931313523, "nll": 0.034914356617148576, "ece10": 0.028228566409425893, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.0198034357787604, "nll": 0.04486109136173615, "ece10": 0.029027648577866882, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 0, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "wilson95": [ 0.7735463743160095, 0.9801879358846942 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 26, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.12279960323790409, "nll": 0.23264934238008977, "ece10": 0.05443245895065126, "calibration_n": 28, "confident_wrong": 1, "macro_f1": 0.9302884615384616, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 1, "inspection": 6, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "wilson95": [ 0.7735463743160095, 0.9801879358846942 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 26, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.11965523568293428, "nll": 0.2569359715924599, "ece10": 0.06674859942541927, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9302884615384616, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 1, "purchase": 6, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 0, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.003129513929033303, "nll": 0.03452111510963976, "ece10": 0.03325763083567961, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "review": { "review": 8, "it": 0, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.008686569726176806, "nll": 0.04283376453744225, "ece10": 0.03942526556581272, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 8, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.010543363238678377, "nll": 0.054012290580363734, "ece10": 0.04966154437396352, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "review": { "review": 8, "it": 0, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 39, "accuracy": 0.975, "wilson95": [ 0.8711863103652591, 0.9955731684973186 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.975, "brier": 0.05465011423890927, "nll": 0.11595690121784867, "ece10": 0.0619307011122391, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9749019607843138, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 7, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 8, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 215, "accuracy": 0.5375, "wilson95": [ 0.4885138311983594, 0.5857727467265683 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 215, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5375, "brier": 0.7329365849890138, "nll": 1.9945333934711968, "ece10": 0.29393089919494436, "calibration_n": 400, "confident_wrong": 66, "macro_f1": 0.44188709037612756, "confusion_matrix": { "sadness": { "sadness": 88, "joy": 8, "love": 5, "anger": 3, "fear": 14, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 23, "joy": 55, "love": 18, "anger": 4, "fear": 13, "surprise": 5, "__failed__": 0 }, "love": { "sadness": 10, "joy": 6, "love": 8, "anger": 3, "fear": 10, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 17, "joy": 4, "love": 4, "anger": 27, "fear": 11, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 10, "joy": 1, "love": 0, "anger": 4, "fear": 35, "surprise": 2, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 2, "love": 1, "anger": 1, "fear": 2, "surprise": 2, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 255, "accuracy": 0.85, "wilson95": [ 0.805182388854671, 0.8859675312881468 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 255, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.21080965527682843, "nll": 0.7705387817513271, "ece10": 0.053393348762999476, "calibration_n": 300, "confident_wrong": 7 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 358, "accuracy": 0.895, "wilson95": [ 0.8611090149132419, 0.921376272562664 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 358, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.895, "brier": 0.1536349840122658, "nll": 0.25957510082087504, "ece10": 0.05916410337857341, "calibration_n": 400, "confident_wrong": 14, "macro_f1": 0.8635654885654886, "confusion_matrix": { "false": { "false": 275, "true": 34, "__failed__": 0 }, "true": { "false": 8, "true": 83, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 331, "accuracy": 0.8275, "wilson95": [ 0.7874046544953168, 0.8613647927157191 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 331, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8275, "brier": 0.29996399914782934, "nll": 0.782180835223517, "ece10": 0.1333159988004034, "calibration_n": 400, "confident_wrong": 47, "macro_f1": 0.8139984952304122, "confusion_matrix": { "world": { "world": 86, "sports": 6, "business": 9, "sci_tech": 2, "__failed__": 0 }, "sports": { "world": 5, "sports": 118, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 5, "sports": 0, "business": 64, "sci_tech": 3, "__failed__": 0 }, "sci_tech": { "world": 12, "sports": 0, "business": 27, "sci_tech": 63, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 348, "accuracy": 0.87, "wilson95": [ 0.8334921544792615, 0.8994687477133592 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 348, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.1764910778052692, "nll": 0.27882865595512113, "ece10": 0.05966438771394891, "calibration_n": 400, "confident_wrong": 12, "macro_f1": 0.860181226640854, "confusion_matrix": { "true": { "true": 121, "false": 32, "__failed__": 0 }, "false": { "true": 20, "false": 227, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 216, "accuracy": 0.72, "wilson95": [ 0.666656114789153, 0.7677809778720466 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 216, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.72, "brier": 0.47151383973354927, "nll": 1.5546578108926037, "ece10": 0.1628026436900568, "calibration_n": 300, "confident_wrong": 36 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 384, "accuracy": 0.96, "wilson95": [ 0.9360177522821589, 0.9752309369302883 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 384, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.96, "brier": 0.0643085593300223, "nll": 0.10818161236350284, "ece10": 0.01679524005203141, "calibration_n": 400, "confident_wrong": 5, "macro_f1": 0.9599639675708138, "confusion_matrix": { "false": { "false": 186, "true": 2, "__failed__": 0 }, "true": { "false": 14, "true": 198, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 293, "accuracy": 0.7325, "wilson95": [ 0.6870591672528669, 0.7735176158816853 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 293, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7325, "brier": 0.3894078931432634, "nll": 0.6393410031583375, "ece10": 0.1600540144160678, "calibration_n": 400, "confident_wrong": 37, "macro_f1": 0.717213629245925, "confusion_matrix": { "false": { "false": 193, "true": 7, "__failed__": 0 }, "true": { "false": 100, "true": 100, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 75, "accuracy": 0.75, "wilson95": [ 0.656955364519384, 0.8245478863771232 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 75, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.38529283133533276, "nll": 0.7438612683959672, "ece10": 0.1801659540197244, "calibration_n": 100, "confident_wrong": 12, "macro_f1": 0.7333333333333334, "confusion_matrix": { "yes": { "yes": 50, "no": 4, "__failed__": 0 }, "no": { "yes": 21, "no": 25, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 46, "accuracy": 0.46, "wilson95": [ 0.36560808309024106, 0.5573513967663177 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 46, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.46, "brier": 0.9020623851486644, "nll": 3.2770197497663305, "ece10": 0.3675403461693489, "calibration_n": 100, "confident_wrong": 31, "macro_f1": 0.5040807027939381, "confusion_matrix": { "balance": { "balance": 3, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 2, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 2, "dispute_charge": 2, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 5, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 3, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 4, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 1, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 4, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 6, "loan_payment": 1, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 4, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 4, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 4, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 1, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 3, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 3, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 144, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.72, "brier": 0.4875230736655765, "nll": 1.0982908368190887, "ece10": 0.21978793557263063, "calibration_n": 200, "confident_wrong": 36 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 132, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.66, "brier": 0.5934876478417422, "nll": 1.6461103587695904, "ece10": 0.23004964220476432, "calibration_n": 200, "confident_wrong": 40 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 74, "accuracy": 0.74, "wilson95": [ 0.6462901055081282, 0.8159530153525186 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 74, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.74, "brier": 0.41786140321735077, "nll": 1.3263220144943597, "ece10": 0.169934565641183, "calibration_n": 100, "confident_wrong": 8, "macro_f1": 0.7009574575751047, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 12, "contract": 0, "resume": 0, "bank_statement": 1, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 5, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 3, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 12, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 2, "clinical_note": 6, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 6, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 1, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 2, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 196, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.8668334100967791, "nll": 2.11168018773198, "ece10": 0.4020414685477456, "calibration_n": 400, "confident_wrong": 123 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 61, "accuracy": 0.61, "wilson95": [ 0.5120301704584073, 0.6998312599360558 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 61, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.6421306719446338, "nll": 1.8508391213912576, "ece10": 0.25589208223881155, "calibration_n": 100, "confident_wrong": 19, "macro_f1": 0.6032620712620712, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 1, "science": 1, "health": 4, "sports": 0, "entertainment": 0, "world": 2, "culture": 1, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 1, "business": 1, "technology": 6, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "science": { "politics": 2, "business": 1, "technology": 0, "science": 6, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 1, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 4, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 2, "weather": 0, "__failed__": 0 }, "world": { "politics": 2, "business": 2, "technology": 1, "science": 0, "health": 1, "sports": 0, "entertainment": 0, "world": 2, "culture": 1, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 8, "weather": 1, "__failed__": 0 }, "weather": { "politics": 0, "business": 2, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 7, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 39, "accuracy": 0.39, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.39, "brier": 0.9374795674504973, "nll": 2.5798993741187526, "ece10": 0.3914903917639607, "calibration_n": 100, "confident_wrong": 18, "macro_f1": 0.3669880311259622, "confusion_matrix": { "biology": { "biology": 8, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 3, "medicine": 7, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 1, "medicine": 0, "chemistry": 2, "physics": 0, "computer_science": 3, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 5, "medicine": 2, "chemistry": 0, "physics": 6, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 2, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 5, "mathematics": 0, "earth_science": 2, "engineering": 3, "economics": 1, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 1, "medicine": 1, "chemistry": 1, "physics": 2, "computer_science": 0, "mathematics": 1, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 3, "medicine": 1, "chemistry": 0, "physics": 1, "computer_science": 2, "mathematics": 0, "earth_science": 4, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 1, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 2, "mathematics": 0, "earth_science": 0, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 2, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 0, "economics": 4, "other": 0, "__failed__": 0 }, "other": { "biology": 4, "medicine": 0, "chemistry": 0, "physics": 2, "computer_science": 1, "mathematics": 0, "earth_science": 2, "engineering": 2, "economics": 1, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 13, "accuracy": 0.13, "wilson95": [ 0.07757167427240512, 0.20980351440076428 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 72, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.36, "brier": 0.830457346744947, "nll": 2.35130920002844, "ece10": 0.40124399457991194, "calibration_n": 100, "confident_wrong": 37 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 29, "accuracy": 0.29, "wilson95": [ 0.2101483574912227, 0.3853889117557111 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 116, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.4022641984022782, "nll": 0.892329067801209, "ece10": 0.17543759025421218, "calibration_n": 100, "confident_wrong": 16 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 83, "accuracy": 0.83, "wilson95": [ 0.7445199523239887, 0.8910643388594006 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 83, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.83, "brier": 0.2615439528841666, "nll": 0.5854972691444545, "ece10": 0.12108781257851155, "calibration_n": 100, "confident_wrong": 10, "macro_f1": 0.830842113860982, "confusion_matrix": { "negative": { "negative": 30, "neutral": 6, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 2, "neutral": 31, "positive": 0, "__failed__": 0 }, "positive": { "negative": 2, "neutral": 7, "positive": 22, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 23, "accuracy": 0.23, "wilson95": [ 0.15843265880303448, 0.3215438302287378 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 110, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.23829778515571098, "nll": 0.43250022538359323, "ece10": 0.08242604229643236, "calibration_n": 100, "confident_wrong": 5 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 45, "accuracy": 0.45, "wilson95": [ 0.3561453797951198, 0.5475539700255787 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 234, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.78, "brier": 0.3737658548208193, "nll": 0.8999884048152552, "ece10": 0.15829446675201245, "calibration_n": 300, "confident_wrong": 39 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 52, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.744484551672765, "nll": 2.8957952298551937, "ece10": 0.24293303399951413, "calibration_n": 100, "confident_wrong": 8, "macro_f1": 0.4589587350091553, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 2, "login_problem": 1, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 2, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 4, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 1, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 3, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 3, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 2, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 1, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 2, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 3, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 5, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 3, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 1, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 38, "accuracy": 0.38, "wilson95": [ 0.29097599252478734, 0.47790244704488916 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.38, "brier": 1.0113218293007, "nll": 3.012661380316303, "ece10": 0.43141709057894806, "calibration_n": 100, "confident_wrong": 28, "macro_f1": 0.3412075185351047, "confusion_matrix": { "billing": { "billing": 7, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 3, "account_access": 7, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 3, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 3, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 3, "account_access": 0, "shipping": 0, "returns": 4, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 2, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 1, "account_access": 2, "shipping": 0, "returns": 2, "technical_outage": 1, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 7, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "compliance": { "billing": 1, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 3, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 5, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 1, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "other": { "billing": 4, "account_access": 1, "shipping": 1, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 11, "accuracy": 0.11, "wilson95": [ 0.06254196357064568, 0.18631296503080313 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 147, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.8416278711129939, "nll": 2.184309358726468, "ece10": 0.36480639827749245, "calibration_n": 300, "confident_wrong": 61 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.6837444639378107, "nll": 2.1082744936885596, "ece10": 0.25077701804863245, "calibration_n": 100, "confident_wrong": 14, "macro_f1": 0.5429828624021001, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 5, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 2, "seat": 0, "baggage": 1, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 7, "car": 0, "visa_docs": 0, "loyalty": 1, "disruption": 2, "other": 1, "__failed__": 0 }, "car": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 1, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 3, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 1, "loyalty": 6, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 5, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 2, "refund": 0, "hotel": 3, "car": 1, "visa_docs": 2, "loyalty": 1, "disruption": 2, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 82, "accuracy": 0.82, "wilson95": [ 0.7333264161369926, 0.8829977450105364 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 82, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.82, "brier": 0.3122271681860343, "nll": 0.8215745190814207, "ece10": 0.12994136017875285, "calibration_n": 100, "confident_wrong": 11, "macro_f1": 0.8026574153234495, "confusion_matrix": { "business": { "business": 24, "science and technology": 0, "sports": 0, "world": 1, "__failed__": 0 }, "science and technology": { "business": 11, "science and technology": 10, "sports": 1, "world": 3, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 2, "science and technology": 0, "sports": 0, "world": 23, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 52, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.606888483472932, "nll": 2.6196352652800625, "ece10": 0.13068263822362497, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.36395502645502653, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 2, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 1, "unrecognised cash withdrawal": 1, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 1, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 2, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 2, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 1, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 1, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 1, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 1, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 1, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 1, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 1, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 44, "accuracy": 0.44, "wilson95": [ 0.34672026630160796, 0.5377189534832302 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 44, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.44, "brier": 0.9172187008448184, "nll": 2.675604959083513, "ece10": 0.3958370736599901, "calibration_n": 100, "confident_wrong": 26, "macro_f1": 0.44154564073918906, "confusion_matrix": { "anger": { "anger": 7, "fear": 3, "joy": 0, "love": 0, "sadness": 7, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 1, "fear": 10, "joy": 0, "love": 0, "sadness": 5, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 8, "love": 0, "sadness": 7, "surprise": 1, "__failed__": 0 }, "love": { "anger": 1, "fear": 0, "joy": 2, "love": 5, "sadness": 9, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 1, "joy": 1, "love": 1, "sadness": 12, "surprise": 1, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 0, "joy": 2, "love": 1, "sadness": 10, "surprise": 2, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2365, "accuracy": 0.8022388059701493, "wilson95": [ 0.7874711828979016, 0.8162197757323552 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2365, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8022388059701493, "brier": 0.30750449033803406, "nll": 1.08790058983228, "ece10": 0.06583691634857214, "calibration_n": 2948, "confident_wrong": 172, "macro_f1": 0.7782579619329416, "confusion_matrix": { "alarm": { "alarm": 92, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "audio": { "alarm": 5, "audio": 45, "calendar": 0, "cooking": 3, "datetime": 0, "email": 0, "general": 4, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "calendar": { "alarm": 7, "audio": 0, "calendar": 350, "cooking": 1, "datetime": 12, "email": 0, "general": 0, "iot": 0, "lists": 4, "music": 0, "news": 2, "play": 0, "qa": 5, "recommendation": 11, "social": 2, "takeaway": 0, "transport": 0, "weather": 6, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 68, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 86, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 12, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 5, "audio": 0, "calendar": 5, "cooking": 6, "datetime": 16, "email": 218, "general": 2, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 12, "recommendation": 0, "social": 2, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "general": { "alarm": 7, "audio": 1, "calendar": 8, "cooking": 24, "datetime": 5, "email": 0, "general": 64, "iot": 0, "lists": 2, "music": 1, "news": 1, "play": 3, "qa": 54, "recommendation": 11, "social": 0, "takeaway": 0, "transport": 2, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 4, "audio": 5, "calendar": 1, "cooking": 15, "datetime": 3, "email": 0, "general": 4, "iot": 178, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 5, "audio": 0, "calendar": 6, "cooking": 1, "datetime": 9, "email": 0, "general": 0, "iot": 1, "lists": 112, "music": 3, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 3, "__failed__": 0 }, "music": { "alarm": 1, "audio": 0, "calendar": 1, "cooking": 6, "datetime": 2, "email": 0, "general": 2, "iot": 0, "lists": 2, "music": 58, "news": 0, "play": 5, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 1, "audio": 0, "calendar": 2, "cooking": 3, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 94, "play": 0, "qa": 22, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "play": { "alarm": 10, "audio": 3, "calendar": 0, "cooking": 18, "datetime": 6, "email": 0, "general": 4, "iot": 2, "lists": 0, "music": 22, "news": 1, "play": 306, "qa": 9, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 4, "__failed__": 0 }, "qa": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 2, "datetime": 1, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 1, "news": 2, "play": 0, "qa": 267, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 1, "weather": 4, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 4, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 4, "play": 5, "qa": 8, "recommendation": 71, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 6, "datetime": 8, "email": 2, "general": 17, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 2, "recommendation": 0, "social": 68, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 10, "datetime": 2, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 3, "social": 0, "takeaway": 39, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 2, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 5, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 110, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 2, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 6, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 139, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2055, "accuracy": 0.6970827679782904, "wilson95": [ 0.6802473125057262, 0.7134052654563154 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2055, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6970827679782904, "brier": 0.44530944945096096, "nll": 1.5342089110262516, "ece10": 0.10870306907199621, "calibration_n": 2948, "confident_wrong": 196, "macro_f1": 0.6793904427991598, "confusion_matrix": { "alarm": { "alarm": 91, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "audio": { "alarm": 9, "audio": 40, "calendar": 0, "cooking": 3, "datetime": 4, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 11, "audio": 0, "calendar": 330, "cooking": 3, "datetime": 22, "email": 0, "general": 1, "iot": 0, "lists": 5, "music": 0, "news": 1, "play": 0, "qa": 4, "recommendation": 12, "social": 1, "takeaway": 0, "transport": 0, "weather": 10, "__failed__": 0 }, "cooking": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 65, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 1, "takeaway": 1, "transport": 0, "weather": 1, "__failed__": 0 }, "datetime": { "alarm": 4, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 86, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 10, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 2, "audio": 10, "calendar": 4, "cooking": 7, "datetime": 22, "email": 207, "general": 3, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 9, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 4, "__failed__": 0 }, "general": { "alarm": 3, "audio": 3, "calendar": 13, "cooking": 22, "datetime": 7, "email": 1, "general": 63, "iot": 1, "lists": 4, "music": 1, "news": 2, "play": 4, "qa": 47, "recommendation": 9, "social": 1, "takeaway": 0, "transport": 3, "weather": 3, "__failed__": 0 }, "iot": { "alarm": 2, "audio": 15, "calendar": 1, "cooking": 18, "datetime": 15, "email": 0, "general": 2, "iot": 156, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 4, "audio": 1, "calendar": 4, "cooking": 1, "datetime": 8, "email": 0, "general": 1, "iot": 0, "lists": 115, "music": 3, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 1, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 1, "audio": 0, "calendar": 1, "cooking": 6, "datetime": 9, "email": 0, "general": 4, "iot": 0, "lists": 2, "music": 53, "news": 0, "play": 2, "qa": 2, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 6, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 4, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 74, "play": 0, "qa": 29, "recommendation": 1, "social": 2, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "play": { "alarm": 13, "audio": 13, "calendar": 3, "cooking": 55, "datetime": 30, "email": 1, "general": 8, "iot": 0, "lists": 2, "music": 81, "news": 1, "play": 150, "qa": 26, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "qa": { "alarm": 3, "audio": 2, "calendar": 1, "cooking": 9, "datetime": 3, "email": 0, "general": 7, "iot": 0, "lists": 1, "music": 0, "news": 2, "play": 0, "qa": 250, "recommendation": 1, "social": 3, "takeaway": 0, "transport": 1, "weather": 4, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 3, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 2, "qa": 9, "recommendation": 74, "social": 0, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "social": { "alarm": 3, "audio": 0, "calendar": 3, "cooking": 4, "datetime": 11, "email": 1, "general": 14, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 5, "recommendation": 0, "social": 62, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "takeaway": { "alarm": 2, "audio": 0, "calendar": 2, "cooking": 10, "datetime": 2, "email": 0, "general": 0, "iot": 1, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 34, "transport": 0, "weather": 1, "__failed__": 0 }, "transport": { "alarm": 4, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 8, "email": 0, "general": 1, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 4, "recommendation": 6, "social": 0, "takeaway": 0, "transport": 90, "weather": 5, "__failed__": 0 }, "weather": { "alarm": 7, "audio": 0, "calendar": 3, "cooking": 5, "datetime": 8, "email": 0, "general": 3, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 10, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 115, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 743, "accuracy": 0.6334185848252344, "wilson95": [ 0.6054487768294435, 0.6605173799999183 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 743, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6334185848252344, "cluster_bootstrap95": [ 0.5979846928923009, 0.6696146144785721 ], "independent_source_groups": 47, "brier": 0.5717980484146448, "nll": 1.1083011414456436, "ece10": 0.22901986126756083, "calibration_n": 1173, "confident_wrong": 161, "macro_f1": 0.6229455627636603, "confusion_matrix": { "negative": { "negative": 151, "neutral": 17, "positive": 14, "__failed__": 0 }, "neutral": { "negative": 196, "neutral": 305, "positive": 89, "__failed__": 0 }, "positive": { "negative": 65, "neutral": 49, "positive": 287, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "jevk5-2b": { "label": "JevK5 2B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 170, "accuracy": 0.85, "wilson95": [ 0.7939442071583334, 0.89286406437758 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 170, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.22206268697294454, "nll": 0.4266309101321071, "ece10": 0.06407487502407584, "calibration_n": 200, "confident_wrong": 7, "macro_f1": 0.8526404313923888, "confusion_matrix": { "world": { "world": 46, "sports": 0, "business": 0, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 5, "sports": 0, "business": 40, "sci_tech": 6, "__failed__": 0 }, "sci_tech": { "world": 3, "sports": 0, "business": 11, "sci_tech": 38, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 124, "accuracy": 0.62, "wilson95": [ 0.5510671801561923, 0.6844099415132637 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 124, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.62, "brier": 0.6038899453466597, "nll": 1.2900374807615373, "ece10": 0.1674299845086823, "calibration_n": 200, "confident_wrong": 13, "macro_f1": 0.4561317040438286, "confusion_matrix": { "sadness": { "sadness": 40, "joy": 3, "love": 2, "anger": 7, "fear": 5, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 13, "joy": 56, "love": 3, "anger": 5, "fear": 2, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 2, "joy": 6, "love": 4, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 8, "joy": 3, "love": 1, "anger": 15, "fear": 1, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 6, "joy": 2, "love": 1, "anger": 1, "fear": 9, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 1, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 141, "accuracy": 0.705, "wilson95": [ 0.6384128547538122, 0.7638605614315084 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 141, "native_tie_decisions_changed": 15, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.705, "brier": 0.4081831905979648, "nll": 0.5980007250725216, "ece10": 0.12088789142585032, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.7049336100622641, "confusion_matrix": { "false": { "false": 72, "true": 37, "__failed__": 0 }, "true": { "false": 22, "true": 69, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 147, "accuracy": 0.735, "wilson95": [ 0.6698242380737758, 0.7913184585289088 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 147, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.735, "brier": 0.33210138635961856, "nll": 0.49858062824244015, "ece10": 0.05003874553327811, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.7344622861294121, "confusion_matrix": { "true": { "true": 69, "false": 8, "__failed__": 0 }, "false": { "true": 45, "false": 78, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3574089842997472, "nll": 0.592140772580268, "ece10": 0.18349438981097835, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.7733333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 5, "accuracy": 0.4166666666666667, "wilson95": [ 0.19326031365875665, 0.6804886874504503 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4166666666666667, "brier": 0.8146175647013223, "nll": 1.4992510210762575, "ece10": 0.334375735939471, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.24, "confusion_matrix": { "maintenance": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.19224560694525328, "nll": 0.2996937705602302, "ece10": 0.15153890250855154, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9142857142857143, "confusion_matrix": { "design": { "design": 2, "inspection": 0, "purchase": 1, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.26990965756846025, "nll": 0.4044975694959961, "ece10": 0.25268911800389526, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.6541666666666667, "confusion_matrix": { "design": { "design": 2, "inspection": 0, "purchase": 1, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 2, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 1, "work": 1, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 85, "accuracy": 0.85, "wilson95": [ 0.7671644040916763, 0.9069401471634337 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 85, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.20576121479385656, "nll": 0.3592058880034207, "ece10": 0.03998630612186804, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.8314418942816768, "confusion_matrix": { "label_000": { "label_000": 25, "label_001": 0, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 9, "label_001": 11, "label_002": 1, "label_003": 4, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 1, "label_003": 24, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 46, "accuracy": 0.46, "wilson95": [ 0.36560808309024106, 0.5573513967663177 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 46, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.46, "brier": 0.662964727960183, "nll": 1.3616745461532582, "ece10": 0.11456441974783634, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.4620039080589347, "confusion_matrix": { "label_000": { "label_000": 8, "label_001": 3, "label_002": 0, "label_003": 1, "label_004": 5, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 9, "label_002": 1, "label_003": 0, "label_004": 6, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 8, "label_003": 5, "label_004": 1, "label_005": 2, "__failed__": 0 }, "label_003": { "label_000": 2, "label_001": 0, "label_002": 2, "label_003": 8, "label_004": 5, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 1, "label_001": 2, "label_002": 3, "label_003": 1, "label_004": 9, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 2, "label_001": 0, "label_002": 5, "label_003": 0, "label_004": 5, "label_005": 4, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 61, "accuracy": 0.61, "wilson95": [ 0.5120301704584073, 0.6998312599360558 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 61, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.5009899565734254, "nll": 1.38774938866473, "ece10": 0.09995903666884727, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.574074074074074, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 1, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 1, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 1, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 1, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 1, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 1, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 1, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 1, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 1, "label_026": 1, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 2, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 1, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 1, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 1, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.006792589351534686, "nll": 0.03778903474807017, "ece10": 0.03524749690328946, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 68, "accuracy": 0.6126126126126126, "wilson95": [ 0.5196688348455415, 0.698022583016307 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 68, "native_tie_decisions_changed": 4, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6126126126126126, "brier": 0.4780346722218381, "nll": 0.8408358681940291, "ece10": 0.07226977705500504, "calibration_n": 111, "confident_wrong": 0 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 57, "accuracy": 0.7916666666666666, "wilson95": [ 0.6843073658064066, 0.869479450178993 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 57, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7916666666666666, "cluster_bootstrap95": [ 0.6805555555555556, 0.9027777777777778 ], "independent_source_groups": 36, "brier": 0.26964375005342833, "nll": 0.4630942926913282, "ece10": 0.07008731019505829, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 16, "accuracy": 0.8421052631578947, "wilson95": [ 0.6243452472967093, 0.9447952836146585 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8421052631578947, "brier": 0.32178383000617794, "nll": 0.5039859826677178, "ece10": 0.20882928859835634, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8403361344537814, "confusion_matrix": { "false": { "false": 9, "true": 2, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.3182454325021592, "nll": 0.49541992715815647, "ece10": 0.16418425453599259, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8056680161943319, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 14, "accuracy": 0.6363636363636364, "wilson95": [ 0.42951708090131013, 0.8026679532963387 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 14, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6363636363636364, "brier": 0.3571813639544574, "nll": 0.5269436837852672, "ece10": 0.25878098459638377, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.6071428571428572, "confusion_matrix": { "false": { "false": 10, "true": 0, "__failed__": 0 }, "true": { "false": 8, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.05330470401996339, "nll": 0.10710253348034425, "ece10": 0.07847103320071377, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9415584415584415, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 3, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 11, "accuracy": 0.6111111111111112, "wilson95": [ 0.3861904160223791, 0.7969475342783638 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6111111111111112, "brier": 0.4435393112688116, "nll": 0.901622921199203, "ece10": 0.20101489215545912, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5466666666666666, "confusion_matrix": { "0": { "0": 1, "1": 2, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.018358626238644076, "nll": 0.06756831455929077, "ece10": 0.059818439118738256, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.32446805240094, "nll": 0.6474726651249435, "ece10": 0.20879710397827952, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5826719576719577, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 1, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "nit": { "bug": 1, "performance": 0, "security": 0, "style": 2, "refactor": 1, "nit": 0, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 9, "accuracy": 0.6, "wilson95": [ 0.357468301206246, 0.8017550385865841 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.48509313535462256, "nll": 0.8380011584938365, "ece10": 0.2918192179428721, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.5414529914529914, "confusion_matrix": { "feature": { "feature": 1, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 2, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 2, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 0, "chore": 1, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 2, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.307621898330002, "nll": 0.48781459514644787, "ece10": 0.19473906426767879, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8235294117647058, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.3503767270388863, "nll": 0.537315973967219, "ece10": 0.25772408313748074, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.875, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.3913054614688633, "nll": 0.7935703447444121, "ece10": 0.14253792963692669, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6333333333333333, "confusion_matrix": { "harassment": { "harassment": 3, "threat": 0, "self_harm": 0, "spam": 0, "none": 2, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 1, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 6, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 17, "accuracy": 0.7727272727272727, "wilson95": [ 0.5656004682477829, 0.8987696001475147 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7727272727272727, "brier": 0.28979790775098396, "nll": 0.6314925762085073, "ece10": 0.11485147335490077, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.7586324786324786, "confusion_matrix": { "news_report": { "news_report": 4, "opinion": 0, "press_release": 1, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 1, "opinion": 4, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 3, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 1, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.019042769149213427, "nll": 0.0649964005921966, "ece10": 0.057257784669529445, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.12220439331360197, "nll": 0.25418130566743125, "ece10": 0.1267303286024736, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "address_issue": { "address_issue": 3, "customs_hold": 1, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.3554931022650576, "nll": 0.5241046510644276, "ece10": 0.19456839389330877, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.575, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03221168339840611, "nll": 0.10656887562891935, "ece10": 0.09241187307242076, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.18461513000507557, "nll": 0.4143116559286179, "ece10": 0.0917807127210746, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.8628571428571428, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 2, "thanks": 0, "fyi": 1, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.20564058149180586, "nll": 0.39679902204839457, "ece10": 0.13211843539378496, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7911111111111111, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 4, "office_supplies": 0, "software": 0, "client_entertainment": 1, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 1, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 11, "accuracy": 0.6111111111111112, "wilson95": [ 0.3861904160223791, 0.7969475342783638 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6111111111111112, "brier": 0.43587899700937965, "nll": 0.6172452063198649, "ece10": 0.20639129824449753, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5418181818181818, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 7, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.666805234809664, "nll": 1.2053587611942957, "ece10": 0.27107068673282314, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.4875, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.42736320240107023, "nll": 0.6860003481087135, "ece10": 0.10713503320646876, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.646031746031746, "confusion_matrix": { "0": { "0": 2, "1": 3, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 3, "2": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 16, "accuracy": 0.8, "wilson95": [ 0.5839825677481064, 0.919342337420202 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.3085695251042451, "nll": 0.6982589702841245, "ece10": 0.20620936251654198, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8102164502164502, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 1, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 2, "other": 1, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 1, "smurfing": 0, "other": 4, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 10, "accuracy": 0.47619047619047616, "wilson95": [ 0.2834400610153417, 0.6763046541715557 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47619047619047616, "brier": 0.7314809678568376, "nll": 1.495903765066576, "ece10": 0.20724963848351352, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.39444444444444443, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 3, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 1, "punctuation": 0, "none": 1, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 1, "none": 3, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.3607684892134725, "nll": 0.49858264934937147, "ece10": 0.22714435423944462, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5151515151515151, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 7, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 22, "accuracy": 0.9565217391304348, "wilson95": [ 0.7900884492974114, 0.9922833338565469 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9565217391304348, "brier": 0.04435885514402764, "nll": 0.12364362299877549, "ece10": 0.09459509371005188, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 1, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 3, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 11, "accuracy": 0.7857142857142857, "wilson95": [ 0.524107694133997, 0.9242861328730683 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.48170655838960197, "nll": 0.9294383307736519, "ece10": 0.2965821706592113, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.6992063492063492, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.561250103516064, "nll": 0.9334647497162833, "ece10": 0.28547947365191306, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.3356643356643357, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 4, "1": 0, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 1, "1": 0, "2": 0, "3": 2, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 6, "accuracy": 0.375, "wilson95": [ 0.18481232558863633, 0.6135895945449727 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.375, "brier": 0.8382562234433206, "nll": 1.6099017778972244, "ece10": 0.3157591566106879, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.29166666666666663, "confusion_matrix": { "0": { "0": 1, "1": 0, "2": 0, "3": 3, "__failed__": 0 }, "1": { "0": 1, "1": 0, "2": 1, "3": 2, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 3, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.3904962363672349, "nll": 0.5792093701772573, "ece10": 0.16216718624612658, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7638888888888888, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.1734072688564996, "nll": 0.44585325100021883, "ece10": 0.09638450997550625, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.86, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 2, "technology": 0, "sports": 1, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.052547338679006704, "nll": 0.14064826429840332, "ece10": 0.1178537428988248, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.3323393498808683, "nll": 0.5087858869025277, "ece10": 0.20333296884160373, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6886446886446886, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.290473096542013, "nll": 0.4759019357932639, "ece10": 0.3749189384120112, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.3202869504864894, "nll": 0.586135791723761, "ece10": 0.1872401449969394, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8193732193732194, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.12266195517274589, "nll": 0.31340511889519584, "ece10": 0.17432549654792912, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8857142857142857, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 1, "french": 3, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 1, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.5844165804651198, "nll": 0.7821491653765742, "ece10": 0.3381966503105268, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.32, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 9, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.30726759182429825, "nll": 0.6272704963279954, "ece10": 0.11462454973569669, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7666666666666666, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 9, "accuracy": 0.5, "wilson95": [ 0.29031021505169974, 0.7096897849483003 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6451838334858414, "nll": 1.365297298134538, "ece10": 0.24646830049264662, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.43714285714285717, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 1, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 2, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 1, "1": 0, "2": 1, "3": 2, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 2, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.45591570732396114, "nll": 0.6444419940145351, "ece10": 0.14864342720029067, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.4920634920634921, "confusion_matrix": { "false": { "false": 5, "true": 5, "__failed__": 0 }, "true": { "false": 3, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 8, "accuracy": 0.5333333333333333, "wilson95": [ 0.30116980025498397, 0.7519046463426261 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5333333333333333, "brier": 0.5076831585965109, "nll": 0.7002592424291247, "ece10": 0.18597349192086435, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.4444444444444444, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.1915889411108307, "nll": 0.36825870308764447, "ece10": 0.1367830286868323, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8732467532467533, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 1, "tech": 0, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.4632269013575989, "nll": 0.6558713177543694, "ece10": 0.09956045738802016, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6357142857142857, "confusion_matrix": { "false": { "false": 4, "true": 4, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 11, "accuracy": 0.5789473684210527, "wilson95": [ 0.3627590353425504, 0.7685810871754576 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5789473684210527, "brier": 0.5088789422822961, "nll": 1.0522002605815892, "ece10": 0.16376128714267144, "calibration_n": 19, "confident_wrong": 1, "macro_f1": 0.36877828054298645, "confusion_matrix": { "0": { "0": 6, "1": 0, "2": 0, "3": 1, "__failed__": 0 }, "1": { "0": 4, "1": 0, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 0, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 6, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.4793938993172204, "nll": 0.6727334642988458, "ece10": 0.10496092551679873, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5096153846153846, "confusion_matrix": { "false": { "false": 10, "true": 1, "__failed__": 0 }, "true": { "false": 5, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.15775072327472014, "nll": 0.3004281717304129, "ece10": 0.24707321963950507, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 13, "accuracy": 0.65, "wilson95": [ 0.43285427668523624, 0.818808175898918 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.5285172670730494, "nll": 0.9477875286257316, "ece10": 0.20865718799378058, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.5666666666666667, "confusion_matrix": { "0": { "0": 6, "1": 0, "2": 0, "3": 1, "__failed__": 0 }, "1": { "0": 2, "1": 1, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 3, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 17, "accuracy": 0.85, "wilson95": [ 0.6395811352592431, 0.9476312541037835 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.13241308749400432, "nll": 0.23480945216988106, "ece10": 0.14610649220915345, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8428571428571429, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 2, "reminder": 0, "music": 0, "small_talk": 2, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 9, "accuracy": 0.45, "wilson95": [ 0.2581978582556243, 0.6579146575496577 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.6412804541759684, "nll": 0.8548663652796524, "ece10": 0.32403282007409906, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.3103448275862069, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 11, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 16, "accuracy": 0.7272727272727273, "wilson95": [ 0.5184826598759762, 0.8684923971201053 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7272727272727273, "brier": 0.28489049006094125, "nll": 0.5579359721847905, "ece10": 0.27443491054822466, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.7256410256410256, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 1, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 2, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 2, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 412, "accuracy": 0.4577777777777778, "wilson95": [ 0.4254765867942454, 0.4904378700407369 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 549, "native_tie_decisions_changed": 29, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5116495806150979, "cluster_bootstrap95": [ 0.4263959390862944, 0.48585707552727697 ], "independent_source_groups": 596, "brier": 0.6052439032956436, "nll": 1.0406205819804357, "ece10": 0.14343993020562493, "calibration_n": 1073, "confident_wrong": 6 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 350, "accuracy": 0.6097560975609756, "wilson95": [ 0.5692466124675816, 0.6488062774326188 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 695, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7425213675213675, "brier": 0.3702375376860531, "nll": 0.7331616789960734, "ece10": 0.03921504187447763, "calibration_n": 936, "confident_wrong": 9 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 221, "accuracy": 0.3157142857142857, "wilson95": [ 0.28236724084641185, 0.35107293695685926 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 491, "native_tie_decisions_changed": 10, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45128676470588236, "cluster_bootstrap95": [ 0.28245363766048504, 0.35014576417514803 ], "independent_source_groups": 650, "brier": 0.6449776036956132, "nll": 1.1913160469774469, "ece10": 0.07593736945208311, "calibration_n": 1088, "confident_wrong": 1 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 18, "accuracy": 0.045, "wilson95": [ 0.028651458653886006, 0.07000472919032381 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 927, "native_tie_decisions_changed": 17, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4635, "brier": 0.6280556295576191, "nll": 1.0897437750227392, "ece10": 0.16694332920534122, "calibration_n": 2000, "confident_wrong": 2 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.044409435725781286, "nll": 0.09565324493011215, "ece10": 0.045718976710212356, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 22, "accuracy": 0.7857142857142857, "wilson95": [ 0.6046141434627812, 0.8978753966710211 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.21407874002207028, "nll": 0.337855628173706, "ece10": 0.11925572464885892, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.7886904761904763, "confusion_matrix": { "work": { "work": 6, "purchase": 0, "inspection": 0, "design": 1, "__failed__": 0 }, "purchase": { "work": 1, "purchase": 5, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 0, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 1, "purchase": 0, "inspection": 1, "design": 5, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.05590470794344603, "nll": 0.12253196557983648, "ece10": 0.07095483508567452, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 21, "accuracy": 0.75, "wilson95": [ 0.5664432968100104, 0.8732350508070668 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3602520152072616, "nll": 0.6326560621667534, "ece10": 0.1333882578366462, "calibration_n": 28, "confident_wrong": 3, "macro_f1": 0.7431932102984734, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 1, "purchase": 5, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 0, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 3, "purchase": 0, "inspection": 1, "design": 3, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 36, "accuracy": 0.9, "wilson95": [ 0.7694822477247771, 0.9604204713173936 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 36, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.1973303212508616, "nll": 0.39114549719745506, "ece10": 0.15485514119434696, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8955368814192344, "confusion_matrix": { "review": { "review": 5, "it": 0, "documents": 0, "procurement": 1, "maintenance": 2, "__failed__": 0 }, "it": { "review": 1, "it": 7, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 37, "accuracy": 0.925, "wilson95": [ 0.8013576647568946, 0.9741639742254119 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.1819703074814446, "nll": 0.459230764429548, "ece10": 0.12943302776869198, "calibration_n": 40, "confident_wrong": 2, "macro_f1": 0.9198592257415787, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 2, "review": 5, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 31, "accuracy": 0.775, "wilson95": [ 0.6249690325768998, 0.8768390867645927 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 31, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.775, "brier": 0.33588583408180206, "nll": 0.639159765111704, "ece10": 0.1694670277224659, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7660130718954248, "confusion_matrix": { "review": { "review": 4, "it": 2, "documents": 1, "procurement": 1, "maintenance": 0, "__failed__": 0 }, "it": { "review": 2, "it": 5, "documents": 0, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 1, "it": 0, "documents": 1, "procurement": 6, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 32, "accuracy": 0.8, "wilson95": [ 0.652426936536005, 0.895000102745623 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 32, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.3422205351002037, "nll": 0.7241168412406979, "ece10": 0.11753420503216512, "calibration_n": 40, "confident_wrong": 1, "macro_f1": 0.7813622291021671, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 6, "documents": 1, "it": 0, "review": 1, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 7, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 1, "documents": 1, "it": 2, "review": 3, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 234, "accuracy": 0.585, "wilson95": [ 0.5361289851942794, 0.6322539247688901 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 234, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.585, "brier": 0.5468553673488984, "nll": 1.1355533743412494, "ece10": 0.08977925571901953, "calibration_n": 400, "confident_wrong": 3, "macro_f1": 0.48856650675914287, "confusion_matrix": { "sadness": { "sadness": 85, "joy": 11, "love": 8, "anger": 7, "fear": 7, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 19, "joy": 79, "love": 11, "anger": 4, "fear": 4, "surprise": 1, "__failed__": 0 }, "love": { "sadness": 10, "joy": 13, "love": 10, "anger": 2, "fear": 2, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 17, "joy": 5, "love": 4, "anger": 32, "fear": 4, "surprise": 2, "__failed__": 0 }, "fear": { "sadness": 17, "joy": 2, "love": 1, "anger": 6, "fear": 26, "surprise": 0, "__failed__": 0 }, "surprise": { "sadness": 1, "joy": 2, "love": 1, "anger": 1, "fear": 3, "surprise": 2, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 249, "accuracy": 0.83, "wilson95": [ 0.7833857374317239, 0.8682699015600757 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 249, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.83, "brier": 0.24719027527885942, "nll": 0.616524743043796, "ece10": 0.06399641747477827, "calibration_n": 300, "confident_wrong": 3 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 356, "accuracy": 0.89, "wilson95": [ 0.8555490895383555, 0.9170313208808931 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 356, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.89, "brier": 0.24281277180309885, "nll": 0.4145024202435234, "ece10": 0.18744876374205535, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.8095238095238095, "confusion_matrix": { "false": { "false": 308, "true": 1, "__failed__": 0 }, "true": { "false": 43, "true": 48, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 338, "accuracy": 0.845, "wilson95": [ 0.8062691036268745, 0.8771674132824607 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 338, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.845, "brier": 0.2371215371155052, "nll": 0.49012551758615375, "ece10": 0.06758355544003708, "calibration_n": 400, "confident_wrong": 23, "macro_f1": 0.832135847982155, "confusion_matrix": { "world": { "world": 88, "sports": 6, "business": 8, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 3, "sports": 120, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 6, "sports": 0, "business": 64, "sci_tech": 2, "__failed__": 0 }, "sci_tech": { "world": 13, "sports": 0, "business": 23, "sci_tech": 66, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 296, "accuracy": 0.74, "wilson95": [ 0.6948757457668063, 0.7805583529527312 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 296, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.74, "brier": 0.33694916845459005, "nll": 0.50264953492149, "ece10": 0.017017498802893057, "calibration_n": 400, "confident_wrong": 2, "macro_f1": 0.7399739973997399, "confusion_matrix": { "true": { "true": 146, "false": 7, "__failed__": 0 }, "false": { "true": 97, "false": 150, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 186, "accuracy": 0.62, "wilson95": [ 0.5638844388564824, 0.6730812480496265 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 186, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.62, "brier": 0.5217627348334559, "nll": 1.300155635370216, "ece10": 0.040093245268185945, "calibration_n": 300, "confident_wrong": 2 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 315, "accuracy": 0.7875, "wilson95": [ 0.7447739406814162, 0.8247564900763633 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 315, "native_tie_decisions_changed": 28, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7875, "brier": 0.379209856621432, "nll": 0.5674304213878262, "ece10": 0.1867724525641711, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.7860437603438403, "confusion_matrix": { "false": { "false": 141, "true": 47, "__failed__": 0 }, "true": { "false": 38, "true": 174, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 220, "accuracy": 0.55, "wilson95": [ 0.5010009985806122, 0.5980477719859582 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 220, "native_tie_decisions_changed": 35, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.4896506600245785, "nll": 0.6833656158891285, "ece10": 0.0313608859562308, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.5071058900846135, "confusion_matrix": { "false": { "false": 169, "true": 31, "__failed__": 0 }, "true": { "false": 149, "true": 51, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 64, "accuracy": 0.64, "wilson95": [ 0.5423540245160874, 0.7272877959859567 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 64, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.64, "brier": 0.4313664190745144, "nll": 0.6203877659239915, "ece10": 0.09269773640348412, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5628946090335114, "confusion_matrix": { "yes": { "yes": 53, "no": 1, "__failed__": 0 }, "no": { "yes": 35, "no": 11, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 54, "accuracy": 0.54, "wilson95": [ 0.44264860323368227, 0.6343919169097589 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 54, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.54, "brier": 0.6320849754512677, "nll": 1.6821629754900917, "ece10": 0.09664533392957858, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5049481617416399, "confusion_matrix": { "balance": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 1, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 6, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 2, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 2, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 5, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 2, "branch_appointment": 2, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 6, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 9, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 4, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 4, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 4, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 5, "other": 0, "__failed__": 0 }, "other": { "balance": 1, "transfer": 1, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 3, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 137, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.685, "brier": 0.3773593211718155, "nll": 0.6525868641692846, "ece10": 0.052257770737582115, "calibration_n": 200, "confident_wrong": 1 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 124, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.62, "brier": 0.508451992202887, "nll": 1.006225273027987, "ece10": 0.0814090756980347, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 79, "accuracy": 0.79, "wilson95": [ 0.7002003116591013, 0.858343459380847 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 79, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.32505259779923906, "nll": 0.8651197944037625, "ece10": 0.10215185104259454, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.7549005274624468, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 14, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 12, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 1, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 6, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 2, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 9, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 5, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 1, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 7, "accuracy": 0.07, "wilson95": [ 0.03431926106727268, 0.137495147390735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 216, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.54, "brier": 0.5976393037859346, "nll": 1.1293047020655802, "ece10": 0.10261523668442343, "calibration_n": 400, "confident_wrong": 5 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 64, "accuracy": 0.64, "wilson95": [ 0.5423540245160874, 0.7272877959859567 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 64, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.64, "brier": 0.5017604730035923, "nll": 1.168279790960544, "ece10": 0.10900174895797915, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6361430236430237, "confusion_matrix": { "politics": { "politics": 3, "business": 0, "technology": 1, "science": 4, "health": 3, "sports": 0, "entertainment": 0, "world": 1, "culture": 1, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 3, "technology": 7, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 1, "business": 1, "technology": 1, "science": 4, "health": 2, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 1, "health": 0, "sports": 7, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 0, "business": 0, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 3, "weather": 1, "__failed__": 0 }, "world": { "politics": 0, "business": 1, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 4, "culture": 0, "weather": 2, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 0, "world": 1, "culture": 8, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 2, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 7, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 36, "accuracy": 0.36, "wilson95": [ 0.2727122040140433, 0.4576459754839126 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 36, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.36, "brier": 0.8049234034193263, "nll": 1.9200121076817807, "ece10": 0.14607505220576808, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.3199457010039454, "confusion_matrix": { "biology": { "biology": 4, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 4, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 0, "medicine": 6, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 3, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 3, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 4, "computer_science": 0, "mathematics": 0, "earth_science": 7, "engineering": 3, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 1, "chemistry": 1, "physics": 0, "computer_science": 7, "mathematics": 0, "earth_science": 3, "engineering": 3, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 2, "mathematics": 0, "earth_science": 2, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 0, "earth_science": 9, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 2, "mathematics": 1, "earth_science": 2, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 3, "engineering": 0, "economics": 4, "other": 0, "__failed__": 0 }, "other": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 1, "mathematics": 0, "earth_science": 6, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 75, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.375, "brier": 0.4702276192379264, "nll": 0.9181507614735511, "ece10": 0.0754132148996083, "calibration_n": 100, "confident_wrong": 2 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 22, "accuracy": 0.22, "wilson95": [ 0.150012823846054, 0.31070353514985793 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 103, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.515, "brier": 0.34277709066190454, "nll": 0.6059934257562538, "ece10": 0.05430308356116996, "calibration_n": 100, "confident_wrong": 4 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 79, "accuracy": 0.79, "wilson95": [ 0.7002003116591013, 0.858343459380847 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 79, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.3057068894932974, "nll": 0.5288750493270541, "ece10": 0.09070615333761042, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.7788132481457003, "confusion_matrix": { "negative": { "negative": 36, "neutral": 0, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 15, "neutral": 16, "positive": 2, "__failed__": 0 }, "positive": { "negative": 2, "neutral": 2, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 14, "accuracy": 0.14, "wilson95": [ 0.08526343659939134, 0.22137188210963837 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 98, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.25272868570349827, "nll": 0.39417270850626346, "ece10": 0.07223477824672979, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 228, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.3297498145388766, "nll": 0.6330239993723125, "ece10": 0.06493621956953288, "calibration_n": 300, "confident_wrong": 12 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 61, "accuracy": 0.61, "wilson95": [ 0.5120301704584073, 0.6998312599360558 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 61, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.544739928270745, "nll": 1.515285290805646, "ece10": 0.09679547124030544, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5263356857894673, "confusion_matrix": { "order_status": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 1, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 1, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 3, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 1, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 4, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 2, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 2, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 1, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 6, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 9, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 4, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 5, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 2, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 1, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 43, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43, "brier": 0.7500971939026083, "nll": 1.8776717308458002, "ece10": 0.10504809287568406, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.4168346643868465, "confusion_matrix": { "billing": { "billing": 3, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 3, "security": 0, "compliance": 1, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 1, "account_access": 8, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 4, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 2, "account_access": 2, "shipping": 0, "returns": 0, "technical_outage": 2, "onboarding": 0, "integrations": 4, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 2, "integrations": 3, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 1, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 6, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 2, "compliance": 2, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 2, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 5, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 4, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "account_access": 0, "shipping": 2, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 3, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 9, "accuracy": 0.09, "wilson95": [ 0.04807254000256516, 0.1622621285271631 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 140, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.686110228963298, "nll": 1.5503126606672935, "ece10": 0.10384271895949342, "calibration_n": 300, "confident_wrong": 0 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 60, "accuracy": 0.6, "wilson95": [ 0.5020025867910618, 0.6905987135675411 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 60, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5553309473723186, "nll": 1.3965153412190603, "ece10": 0.08429543179435511, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5699884480844233, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 1, "change_flight": 5, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 1, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 11, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 1, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 4, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 0, "loyalty": 6, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 3, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 4, "car": 2, "visa_docs": 2, "loyalty": 1, "disruption": 1, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 86, "accuracy": 0.86, "wilson95": [ 0.7786281178903616, 0.9147365634006086 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.17634375158563947, "nll": 0.3202389737498377, "ece10": 0.060074691094638845, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8576054167486392, "confusion_matrix": { "business": { "business": 20, "science and technology": 3, "sports": 0, "world": 2, "__failed__": 0 }, "science and technology": { "business": 5, "science and technology": 18, "sports": 1, "world": 1, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 1, "science and technology": 0, "sports": 1, "world": 23, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 60, "accuracy": 0.6, "wilson95": [ 0.5020025867910618, 0.6905987135675411 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 60, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5209810795099841, "nll": 1.4677242972640594, "ece10": 0.08828076215292933, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5556216931216931, "confusion_matrix": { "refund not showing up": { "refund not showing up": 1, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 1, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 1, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 1, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 1, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 1, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 1, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 1, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 1, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 2, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 1, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 2, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 1, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 1, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 1, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 1, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 1, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 1, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 1, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 50, "accuracy": 0.5, "wilson95": [ 0.4038315303659956, 0.5961684696340044 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 50, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6561633186376706, "nll": 1.370364692975999, "ece10": 0.11367704590552188, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.5006739838261578, "confusion_matrix": { "anger": { "anger": 7, "fear": 4, "joy": 0, "love": 1, "sadness": 5, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 0, "fear": 12, "joy": 1, "love": 0, "sadness": 4, "surprise": 0, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 11, "love": 2, "sadness": 1, "surprise": 2, "__failed__": 0 }, "love": { "anger": 1, "fear": 0, "joy": 3, "love": 6, "sadness": 7, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 2, "joy": 4, "love": 1, "sadness": 9, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 2, "fear": 0, "joy": 3, "love": 0, "sadness": 6, "surprise": 5, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2089, "accuracy": 0.7086160108548168, "wilson95": [ 0.6919499745020325, 0.7247390710171682 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2089, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7086160108548168, "brier": 0.3951722257707062, "nll": 0.9480693622260249, "ece10": 0.03122218377772786, "calibration_n": 2948, "confident_wrong": 38, "macro_f1": 0.7331450627822775, "confusion_matrix": { "alarm": { "alarm": 75, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 7, "email": 0, "general": 2, "iot": 4, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 33, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 20, "iot": 6, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 0, "audio": 0, "calendar": 310, "cooking": 0, "datetime": 34, "email": 1, "general": 29, "iot": 0, "lists": 5, "music": 0, "news": 3, "play": 0, "qa": 2, "recommendation": 9, "social": 1, "takeaway": 4, "transport": 1, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 66, "datetime": 0, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 97, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 0, "datetime": 1, "email": 211, "general": 33, "iot": 0, "lists": 13, "music": 0, "news": 1, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 2, "weather": 2, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 7, "cooking": 7, "datetime": 4, "email": 0, "general": 127, "iot": 3, "lists": 1, "music": 1, "news": 0, "play": 1, "qa": 13, "recommendation": 12, "social": 1, "takeaway": 1, "transport": 5, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 7, "calendar": 0, "cooking": 5, "datetime": 2, "email": 0, "general": 30, "iot": 168, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 0, "email": 0, "general": 12, "iot": 3, "lists": 121, "music": 3, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 9, "iot": 0, "lists": 1, "music": 69, "news": 0, "play": 1, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 27, "iot": 0, "lists": 0, "music": 0, "news": 90, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 1, "weather": 1, "__failed__": 0 }, "play": { "alarm": 0, "audio": 4, "calendar": 0, "cooking": 0, "datetime": 3, "email": 1, "general": 61, "iot": 3, "lists": 1, "music": 108, "news": 1, "play": 203, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 10, "email": 1, "general": 123, "iot": 1, "lists": 0, "music": 2, "news": 3, "play": 1, "qa": 127, "recommendation": 7, "social": 0, "takeaway": 1, "transport": 6, "weather": 5, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 17, "iot": 0, "lists": 1, "music": 0, "news": 2, "play": 0, "qa": 1, "recommendation": 70, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 0, "email": 1, "general": 49, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 49, "takeaway": 3, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 3, "email": 0, "general": 6, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 46, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 4, "email": 0, "general": 6, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 109, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 15, "iot": 5, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 12, "weather": 118, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1682, "accuracy": 0.5705563093622795, "wilson95": [ 0.5526074401050074, 0.5883215378917304 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1682, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5705563093622795, "brier": 0.5627228643486566, "nll": 1.406248733060295, "ece10": 0.10273055937424116, "calibration_n": 2948, "confident_wrong": 47, "macro_f1": 0.615479401031398, "confusion_matrix": { "alarm": { "alarm": 78, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 9, "email": 0, "general": 3, "iot": 4, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 32, "calendar": 0, "cooking": 0, "datetime": 3, "email": 0, "general": 21, "iot": 2, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 1, "audio": 0, "calendar": 272, "cooking": 0, "datetime": 49, "email": 1, "general": 50, "iot": 4, "lists": 6, "music": 1, "news": 1, "play": 0, "qa": 0, "recommendation": 8, "social": 1, "takeaway": 5, "transport": 1, "weather": 0, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 60, "datetime": 1, "email": 0, "general": 8, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 96, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 1, "email": 207, "general": 48, "iot": 0, "lists": 6, "music": 0, "news": 1, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 2, "weather": 0, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 6, "cooking": 4, "datetime": 12, "email": 1, "general": 134, "iot": 2, "lists": 0, "music": 1, "news": 0, "play": 2, "qa": 4, "recommendation": 10, "social": 0, "takeaway": 0, "transport": 5, "weather": 5, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 14, "calendar": 0, "cooking": 3, "datetime": 4, "email": 0, "general": 46, "iot": 142, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 4, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 1, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 18, "iot": 2, "lists": 114, "music": 2, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 12, "iot": 0, "lists": 0, "music": 66, "news": 0, "play": 1, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 5, "email": 1, "general": 50, "iot": 1, "lists": 1, "music": 0, "news": 57, "play": 0, "qa": 1, "recommendation": 3, "social": 0, "takeaway": 1, "transport": 2, "weather": 2, "__failed__": 0 }, "play": { "alarm": 0, "audio": 13, "calendar": 0, "cooking": 0, "datetime": 6, "email": 0, "general": 130, "iot": 4, "lists": 4, "music": 152, "news": 4, "play": 65, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 3, "weather": 1, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 1, "datetime": 21, "email": 0, "general": 186, "iot": 4, "lists": 1, "music": 2, "news": 1, "play": 0, "qa": 50, "recommendation": 8, "social": 0, "takeaway": 0, "transport": 4, "weather": 7, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 7, "cooking": 1, "datetime": 1, "email": 0, "general": 21, "iot": 0, "lists": 0, "music": 3, "news": 1, "play": 0, "qa": 0, "recommendation": 53, "social": 0, "takeaway": 1, "transport": 6, "weather": 0, "__failed__": 0 }, "social": { "alarm": 1, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 54, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 43, "takeaway": 4, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 3, "email": 0, "general": 8, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 42, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 9, "email": 0, "general": 10, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 4, "social": 0, "takeaway": 1, "transport": 95, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 14, "email": 0, "general": 28, "iot": 6, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 26, "weather": 76, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 612, "accuracy": 0.5217391304347826, "wilson95": [ 0.49312845752650364, 0.5502078811262039 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 612, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5217391304347826, "cluster_bootstrap95": [ 0.491277341213535, 0.5507027394873084 ], "independent_source_groups": 47, "brier": 0.6568269314870445, "nll": 1.1666194292143734, "ece10": 0.21769645082168626, "calibration_n": 1173, "confident_wrong": 115, "macro_f1": 0.5084765899811257, "confusion_matrix": { "negative": { "negative": 119, "neutral": 26, "positive": 37, "__failed__": 0 }, "neutral": { "negative": 206, "neutral": 210, "positive": 174, "__failed__": 0 }, "positive": { "negative": 70, "neutral": 48, "positive": 283, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "jevk5": { "label": "JevK5 4B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 173, "accuracy": 0.865, "wilson95": [ 0.8107086162924745, 0.9055342954521209 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 173, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.865, "brier": 0.20629990338985557, "nll": 0.38851065719362987, "ece10": 0.04572574949293466, "calibration_n": 200, "confident_wrong": 7, "macro_f1": 0.8675354674801985, "confusion_matrix": { "world": { "world": 45, "sports": 0, "business": 2, "sci_tech": 3, "__failed__": 0 }, "sports": { "world": 0, "sports": 45, "business": 2, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 48, "sci_tech": 1, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 15, "sci_tech": 35, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 121, "accuracy": 0.605, "wilson95": [ 0.5358834915928536, 0.6701589898679203 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 121, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.605, "brier": 0.6165565116393024, "nll": 1.3337066406161557, "ece10": 0.19142312413681856, "calibration_n": 200, "confident_wrong": 22, "macro_f1": 0.5103769772947631, "confusion_matrix": { "sadness": { "sadness": 42, "joy": 3, "love": 2, "anger": 5, "fear": 5, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 14, "joy": 48, "love": 9, "anger": 5, "fear": 3, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 3, "joy": 4, "love": 5, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 8, "joy": 3, "love": 1, "anger": 15, "fear": 1, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 5, "joy": 2, "love": 1, "anger": 1, "fear": 10, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 166, "accuracy": 0.83, "wilson95": [ 0.7718411637168737, 0.8757209208741302 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 166, "native_tie_decisions_changed": 3, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.83, "brier": 0.2053969158436986, "nll": 0.317333428883906, "ece10": 0.03500542573722781, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.8176748176748176, "confusion_matrix": { "false": { "false": 109, "true": 0, "__failed__": 0 }, "true": { "false": 34, "true": 57, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 180, "accuracy": 0.9, "wilson95": [ 0.8505941875672826, 0.9343295513309041 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 180, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.15426469502091616, "nll": 0.26791065535431247, "ece10": 0.049008744956347994, "calibration_n": 200, "confident_wrong": 9, "macro_f1": 0.8885918003565062, "confusion_matrix": { "true": { "true": 58, "false": 19, "__failed__": 0 }, "false": { "true": 1, "false": 122, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.19406158979080823, "nll": 0.36433180338496746, "ece10": 0.14743345177624315, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8533333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3866814044408296, "nll": 0.8110843181290925, "ece10": 0.2468632356452755, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.780952380952381, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.010665529125141673, "nll": 0.04605295625450847, "ece10": 0.04278610373631195, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.004497116273759279, "nll": 0.033897777710700856, "ece10": 0.03244997734084652, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 86, "accuracy": 0.86, "wilson95": [ 0.7786281178903616, 0.9147365634006086 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.18308041578904066, "nll": 0.3105711298700485, "ece10": 0.07913129270570944, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8431784651649648, "confusion_matrix": { "label_000": { "label_000": 25, "label_001": 0, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 11, "label_001": 11, "label_002": 1, "label_003": 2, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 49, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.6823550161267901, "nll": 1.4802894813664196, "ece10": 0.17071480954222973, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.47032227032227025, "confusion_matrix": { "label_000": { "label_000": 12, "label_001": 2, "label_002": 1, "label_003": 1, "label_004": 1, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 2, "label_001": 11, "label_002": 1, "label_003": 0, "label_004": 2, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 10, "label_003": 2, "label_004": 2, "label_005": 2, "__failed__": 0 }, "label_003": { "label_000": 4, "label_001": 1, "label_002": 4, "label_003": 5, "label_004": 3, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 2, "label_001": 2, "label_002": 4, "label_003": 0, "label_004": 8, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 2, "label_001": 2, "label_002": 5, "label_003": 0, "label_004": 4, "label_005": 3, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 76, "accuracy": 0.76, "wilson95": [ 0.6676766365018553, 0.8330867444305122 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 76, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.35391155582453265, "nll": 0.8811095231586811, "ece10": 0.07854334857331219, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.7421296296296295, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 1, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 2, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 1, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 2, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 1, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 1, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.002564258245019974, "nll": 0.018909233882413556, "ece10": 0.01818560369925557, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 86, "accuracy": 0.7747747747747747, "wilson95": [ 0.6886323576918694, 0.8425347018910412 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 86, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7747747747747747, "brier": 0.3126583874168679, "nll": 0.550806130074542, "ece10": 0.06728756782445458, "calibration_n": 111, "confident_wrong": 3 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 68, "accuracy": 0.9444444444444444, "wilson95": [ 0.8656798403400208, 0.9781857840186834 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 68, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "cluster_bootstrap95": [ 0.875, 1.0 ], "independent_source_groups": 36, "brier": 0.10897157825737197, "nll": 0.21548754226073286, "ece10": 0.057884710842761145, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 17, "accuracy": 0.8947368421052632, "wilson95": [ 0.6860591728889099, 0.9706414397011299 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 17, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8947368421052632, "brier": 0.10977208456589532, "nll": 0.2090099048712497, "ece10": 0.12528611287631314, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8869047619047619, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.13780209586037648, "nll": 0.2280407379490552, "ece10": 0.13945636104456066, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9372549019607843, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.22884026949837102, "nll": 0.3664375949103333, "ece10": 0.12572266863438294, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8562091503267973, "confusion_matrix": { "false": { "false": 7, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.011126380800023, "nll": 0.041910417388036886, "ece10": 0.03842860514567331, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.1561744346689867, "nll": 0.3206335380315653, "ece10": 0.18362668778821406, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9377777777777776, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.0421357152688787, "nll": 0.0690492870259822, "ece10": 0.04944566299488352, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 1, "roads": 3, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.2535992254075804, "nll": 0.41793171818733477, "ece10": 0.1635544091404896, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7444444444444445, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 1, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.198177597256195, "nll": 0.4524981904126705, "ece10": 0.11491490195365797, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8277777777777778, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 1, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04959620851071446, "nll": 0.14756130540129297, "ece10": 0.13270098252109605, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.23181899026498373, "nll": 0.3768255907179151, "ece10": 0.12969039339199218, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8117647058823529, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.25399224725065195, "nll": 0.44581330530356045, "ece10": 0.059709210096466125, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8427350427350427, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 1, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 1, "none": 5, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.21360980158281845, "nll": 0.3835136453728099, "ece10": 0.10143398438669775, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8565656565656565, "confusion_matrix": { "news_report": { "news_report": 4, "opinion": 0, "press_release": 1, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 2, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 2, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00624963104097276, "nll": 0.020737171076565995, "ece10": 0.018787103042899884, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00017425635130529794, "nll": 0.007921278295745437, "ece10": 0.007860784516176035, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.09146597416222225, "nll": 0.21203579519411409, "ece10": 0.1825925820601922, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0009555116526070991, "nll": 0.012853109739635027, "ece10": 0.01249812568035431, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.12005092828942905, "nll": 0.2318802937240194, "ece10": 0.08435934777549142, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.04893029698011937, "nll": 0.10947968449737994, "ece10": 0.08921389671480372, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.1561989409894204, "nll": 0.27198931327698256, "ece10": 0.09506126026090003, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8875, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.4381577620392558, "nll": 0.6269918497574919, "ece10": 0.23816848939191534, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5732142857142857, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 5, "3": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.1936042666560687, "nll": 0.3468922468894355, "ece10": 0.1190701898132398, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.875, "confusion_matrix": { "0": { "0": 3, "1": 2, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 17, "accuracy": 0.85, "wilson95": [ 0.6395811352592431, 0.9476312541037835 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.18990268674427985, "nll": 0.35977006450040944, "ece10": 0.12559562206480876, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8425396825396826, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 4, "smurfing": 0, "other": 0, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 1, "smurfing": 2, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 1, "other": 4, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 12, "accuracy": 0.5714285714285714, "wilson95": [ 0.3654655436930671, 0.7553003107462404 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5714285714285714, "brier": 0.6776341114824941, "nll": 1.4643661767063196, "ece10": 0.2584011432498143, "calibration_n": 21, "confident_wrong": 1, "macro_f1": 0.5599999999999999, "confusion_matrix": { "spelling": { "spelling": 1, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 2, "none": 0, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 3, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 1, "none": 2, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 2, "none": 3, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02524785651768292, "nll": 0.10218424825731164, "ece10": 0.09495481874315824, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0009691011145495029, "nll": 0.01636014229065502, "ece10": 0.01602747362608397, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 11, "accuracy": 0.7857142857142857, "wilson95": [ 0.524107694133997, 0.9242861328730683 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.36831464843380646, "nll": 0.6665999817169208, "ece10": 0.1255209795381149, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.6433333333333333, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.18722735142891264, "nll": 0.353040041380874, "ece10": 0.24379009281462416, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9444444444444444, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.3823673960991346, "nll": 0.6671060655897724, "ece10": 0.07751830239989718, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6761904761904762, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 1, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 2, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.3352657674405877, "nll": 0.49307318380172627, "ece10": 0.2555246258839885, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7424242424242424, "confusion_matrix": { "false": { "false": 4, "true": 4, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.013993366721183937, "nll": 0.05729555494396921, "ece10": 0.051965732517092335, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02740810820299186, "nll": 0.09540828949558287, "ece10": 0.08520003006252976, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0798056720880044, "nll": 0.19962279953505047, "ece10": 0.17531551595803938, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.10540466504592466, "nll": 0.22326669321013623, "ece10": 0.1880508906598867, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.18050790114701615, "nll": 0.3289948981524173, "ece10": 0.22477693180995784, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8119658119658119, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 3, "2": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.08674386550659373, "nll": 0.20280494267112442, "ece10": 0.05910191632683096, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571427, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.3259999532978739, "nll": 0.4858398217532051, "ece10": 0.24071001048563898, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8819444444444444, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.056191321280212314, "nll": 0.11895337023590945, "ece10": 0.09365969898488263, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 11, "accuracy": 0.6111111111111112, "wilson95": [ 0.3861904160223791, 0.7969475342783638 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6111111111111112, "brier": 0.45517836858140487, "nll": 0.9406949522584457, "ece10": 0.3122574871877278, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 1, "3": 2, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.23819143759945796, "nll": 0.3731806137161593, "ece10": 0.21635730135845405, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.746031746031746, "confusion_matrix": { "false": { "false": 7, "true": 3, "__failed__": 0 }, "true": { "false": 1, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.19864424400196443, "nll": 0.33349631332152196, "ece10": 0.11864950422675312, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8660714285714286, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.06230451430558334, "nll": 0.13250594153846187, "ece10": 0.10622571012772747, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 1, "tech": 0, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.12389519809026008, "nll": 0.2616268600727551, "ece10": 0.22115335174956996, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 13, "accuracy": 0.6842105263157895, "wilson95": [ 0.46010407675327475, 0.8463562091220773 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6842105263157895, "brier": 0.4406244651339904, "nll": 0.759154984971786, "ece10": 0.1750448895055363, "calibration_n": 19, "confident_wrong": 1, "macro_f1": 0.5556318681318682, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.33062214942146884, "nll": 0.49964005249959187, "ece10": 0.1208795438400162, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5502645502645502, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03767221499763999, "nll": 0.11609458387407987, "ece10": 0.10479470284304923, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 14, "accuracy": 0.7, "wilson95": [ 0.4810271816464765, 0.8545227551323956 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.43066169337065097, "nll": 0.7154081923801104, "ece10": 0.1673683422354712, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.6767857142857143, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 2, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.035799314235940835, "nll": 0.07201047889434202, "ece10": 0.053394935281089656, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 16, "accuracy": 0.8, "wilson95": [ 0.5839825677481064, 0.919342337420202 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.3021805593475881, "nll": 0.4727025193815721, "ece10": 0.10683368715265362, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.797979797979798, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.1953838443540386, "nll": 0.37053981589999146, "ece10": 0.1640795578371113, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8683261183261183, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 2, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 530, "accuracy": 0.5888888888888889, "wilson95": [ 0.5564315406812905, 0.6205906554555887 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 683, "native_tie_decisions_changed": 19, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6365330848089469, "cluster_bootstrap95": [ 0.5571417931621507, 0.6284686234340358 ], "independent_source_groups": 596, "brier": 0.4516684685894945, "nll": 0.8283512865899687, "ece10": 0.1215362579834944, "calibration_n": 1073, "confident_wrong": 24 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 417, "accuracy": 0.7264808362369338, "wilson95": [ 0.6885986103910212, 0.761351797339551 ], "valid": 573, "failures": 1, "coverage": 0.9982578397212544, "heads": 936, "head_correct": 770, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8226495726495726, "brier": 0.2566945807665706, "nll": 0.4947648742928464, "ece10": 0.03256240506414886, "calibration_n": 934, "confident_wrong": 26 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 333, "accuracy": 0.4757142857142857, "wilson95": [ 0.4389515730016587, 0.512742093840633 ], "valid": 658, "failures": 42, "coverage": 0.94, "heads": 1088, "head_correct": 621, "native_tie_decisions_changed": 2, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5707720588235294, "cluster_bootstrap95": [ 0.43829588530202984, 0.5120584575226879 ], "independent_source_groups": 650, "brier": 0.5091915675429386, "nll": 0.9594583270413226, "ece10": 0.06776697223637727, "calibration_n": 1004, "confident_wrong": 21 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 55, "accuracy": 0.1375, "wilson95": [ 0.10718448673484568, 0.1747119266575193 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1258, "native_tie_decisions_changed": 14, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.629, "brier": 0.4892236085584269, "nll": 0.8448860568298443, "ece10": 0.08987828342589538, "calibration_n": 2000, "confident_wrong": 54 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.03509632178095702, "nll": 0.07018469621114734, "ece10": 0.05383255488559251, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 6, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.024797312694949093, "nll": 0.06009470354612155, "ece10": 0.04945190617477637, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 0, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.04878739136175675, "nll": 0.10641803148195761, "ece10": 0.05367815144675577, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 6, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.04918380256653287, "nll": 0.10645493800202492, "ece10": 0.08338521020512406, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 1, "purchase": 0, "inspection": 0, "design": 6, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.1479415353669936, "nll": 0.2843316930301508, "ece10": 0.07419463217010605, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8666666666666666, "confusion_matrix": { "review": { "review": 4, "it": 0, "documents": 3, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 7, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 37, "accuracy": 0.925, "wilson95": [ 0.8013576647568946, 0.9741639742254119 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.08542328260692533, "nll": 0.18245216594058053, "ece10": 0.12218175466877, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9243697478991596, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 7, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 6, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 32, "accuracy": 0.8, "wilson95": [ 0.652426936536005, 0.895000102745623 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 32, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.23186739803405612, "nll": 0.41858844038283066, "ece10": 0.12241920970449986, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7454575163398692, "confusion_matrix": { "review": { "review": 1, "it": 1, "documents": 2, "procurement": 1, "maintenance": 3, "__failed__": 0 }, "it": { "review": 0, "it": 7, "documents": 0, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 36, "accuracy": 0.9, "wilson95": [ 0.7694822477247771, 0.9604204713173936 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 36, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.1424522470652882, "nll": 0.29112199186875254, "ece10": 0.15375216559401084, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.888888888888889, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 0, "documents": 2, "it": 0, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 240, "accuracy": 0.6, "wilson95": [ 0.5512590700269542, 0.6468384711061865 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 240, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5714247610976728, "nll": 1.196105717515645, "ece10": 0.15958541215227084, "calibration_n": 400, "confident_wrong": 20, "macro_f1": 0.5234083457812474, "confusion_matrix": { "sadness": { "sadness": 92, "joy": 10, "love": 6, "anger": 3, "fear": 8, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 26, "joy": 72, "love": 14, "anger": 2, "fear": 2, "surprise": 2, "__failed__": 0 }, "love": { "sadness": 10, "joy": 9, "love": 13, "anger": 2, "fear": 2, "surprise": 1, "__failed__": 0 }, "anger": { "sadness": 20, "joy": 3, "love": 4, "anger": 34, "fear": 1, "surprise": 2, "__failed__": 0 }, "fear": { "sadness": 20, "joy": 1, "love": 1, "anger": 2, "fear": 26, "surprise": 2, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 1, "love": 1, "anger": 1, "fear": 2, "surprise": 3, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 273, "accuracy": 0.91, "wilson95": [ 0.8722229892013517, 0.9374097743945206 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 273, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.91, "brier": 0.12449674203466804, "nll": 0.30635216997222575, "ece10": 0.05804636376102876, "calibration_n": 300, "confident_wrong": 1 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 356, "accuracy": 0.89, "wilson95": [ 0.8555490895383555, 0.9170313208808931 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 356, "native_tie_decisions_changed": 5, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.89, "brier": 0.16454072920010054, "nll": 0.27493550264489264, "ece10": 0.038426945965342, "calibration_n": 400, "confident_wrong": 4, "macro_f1": 0.8513312609812136, "confusion_matrix": { "false": { "false": 280, "true": 29, "__failed__": 0 }, "true": { "false": 15, "true": 76, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 333, "accuracy": 0.8325, "wilson95": [ 0.7927798983351753, 0.8658944259325176 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 333, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8325, "brier": 0.26476429535201523, "nll": 0.5266933747666447, "ece10": 0.0947449559168019, "calibration_n": 400, "confident_wrong": 30, "macro_f1": 0.8157455564670495, "confusion_matrix": { "world": { "world": 87, "sports": 6, "business": 9, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 2, "sports": 121, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 3, "sports": 0, "business": 69, "sci_tech": 0, "__failed__": 0 }, "sci_tech": { "world": 12, "sports": 0, "business": 34, "sci_tech": 56, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 354, "accuracy": 0.885, "wilson95": [ 0.850008455984224, 0.9126670773783676 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 354, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.885, "brier": 0.17585066238288086, "nll": 0.2841964350222611, "ece10": 0.04073970717731667, "calibration_n": 400, "confident_wrong": 7, "macro_f1": 0.8694628110899856, "confusion_matrix": { "true": { "true": 108, "false": 45, "__failed__": 0 }, "false": { "true": 1, "false": 246, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 229, "accuracy": 0.7633333333333333, "wilson95": [ 0.7120967390716028, 0.8079112960834695 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 229, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7633333333333333, "brier": 0.3719811402146598, "nll": 0.9550936453461017, "ece10": 0.06765823781769013, "calibration_n": 300, "confident_wrong": 1 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 337, "accuracy": 0.8425, "wilson95": [ 0.8035652075632727, 0.8749188708177342 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 337, "native_tie_decisions_changed": 7, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8425, "brier": 0.18538110581489098, "nll": 0.2971073745050602, "ece10": 0.060840538506881814, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.8411407606331676, "confusion_matrix": { "false": { "false": 187, "true": 1, "__failed__": 0 }, "true": { "false": 62, "true": 150, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 264, "accuracy": 0.66, "wilson95": [ 0.6122516850900368, 0.7047043807229885 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 264, "native_tie_decisions_changed": 6, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.66, "brier": 0.4320109153874303, "nll": 0.614891381649825, "ece10": 0.13947471124163038, "calibration_n": 400, "confident_wrong": 10, "macro_f1": 0.6170200782856017, "confusion_matrix": { "false": { "false": 199, "true": 1, "__failed__": 0 }, "true": { "false": 135, "true": 65, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 70, "accuracy": 0.7, "wilson95": [ 0.6041514536665332, 0.7810511470506724 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 70, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.5032115561016561, "nll": 0.8689256484409492, "ece10": 0.21411079756429385, "calibration_n": 100, "confident_wrong": 20, "macro_f1": 0.6493688639551192, "confusion_matrix": { "yes": { "yes": 54, "no": 0, "__failed__": 0 }, "no": { "yes": 30, "no": 16, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 59, "accuracy": 0.59, "wilson95": [ 0.4920143005341078, 0.6813268697886348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 59, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.59, "brier": 0.6454847854643639, "nll": 1.632616362676634, "ece10": 0.17331371688069475, "calibration_n": 100, "confident_wrong": 7, "macro_f1": 0.5855620074370075, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 4, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 7, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 2, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 6, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 3, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 5, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 8, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 5, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 4, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 1, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 144, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.72, "brier": 0.40256692191671595, "nll": 0.6744774180617585, "ece10": 0.14543630618726694, "calibration_n": 200, "confident_wrong": 6 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 46, "accuracy": 0.46, "wilson95": [ 0.36560808309024106, 0.5573513967663177 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 131, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.655, "brier": 0.469361334768084, "nll": 1.004987567053491, "ece10": 0.08992630650333529, "calibration_n": 200, "confident_wrong": 6 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.3130913873789116, "nll": 0.8858668883670768, "ece10": 0.11492976020209333, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.7416087652392954, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 14, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 12, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 1, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 7, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 2, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 9, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 2, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 6, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 1, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 10, "accuracy": 0.1, "wilson95": [ 0.0552291370606751, 0.17436566150491345 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 222, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.555, "brier": 0.6925261326776914, "nll": 1.539966389967807, "ece10": 0.2574938703248928, "calibration_n": 400, "confident_wrong": 62 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 63, "accuracy": 0.63, "wilson95": [ 0.5322052958094291, 0.7181763946567546 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 63, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.63, "brier": 0.5164103073700824, "nll": 1.3290486988692425, "ece10": 0.13492103149740173, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.622919045918147, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "science": 3, "health": 5, "sports": 0, "entertainment": 1, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 10, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 3, "technology": 7, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 2, "technology": 1, "science": 5, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 6, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 0, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 7, "world": 0, "culture": 1, "weather": 0, "__failed__": 0 }, "world": { "politics": 0, "business": 5, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 2, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 1, "world": 0, "culture": 7, "weather": 1, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 2, "culture": 0, "weather": 5, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 37, "accuracy": 0.37, "wilson95": [ 0.28182360534324524, 0.4677947041905709 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.37, "brier": 0.8891345886389993, "nll": 2.307800894358801, "ece10": 0.30491165188244057, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.3478077856096432, "confusion_matrix": { "biology": { "biology": 6, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 9, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 3, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 2, "medicine": 4, "chemistry": 0, "physics": 4, "computer_science": 0, "mathematics": 0, "earth_science": 4, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 1, "medicine": 3, "chemistry": 2, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 5, "engineering": 2, "economics": 1, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 1, "earth_science": 2, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 0, "medicine": 4, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 7, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 1, "engineering": 3, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 0, "economics": 3, "other": 0, "__failed__": 0 }, "other": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 0, "earth_science": 4, "engineering": 4, "economics": 1, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 12, "accuracy": 0.12, "wilson95": [ 0.06999406437019488, 0.19812099426711421 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 82, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.41, "brier": 0.5367946532090028, "nll": 1.2062028270801313, "ece10": 0.19404504689001045, "calibration_n": 100, "confident_wrong": 14 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 33, "accuracy": 0.33, "wilson95": [ 0.24563122734128034, 0.4269465620490947 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 125, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.2660684929810888, "nll": 0.4777374557870292, "ece10": 0.05987029321738507, "calibration_n": 100, "confident_wrong": 3 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 86, "accuracy": 0.86, "wilson95": [ 0.7786281178903616, 0.9147365634006086 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.2055576054624407, "nll": 0.39451984569036147, "ece10": 0.0502065621279, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.8586091883606901, "confusion_matrix": { "negative": { "negative": 36, "neutral": 0, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 9, "neutral": 23, "positive": 1, "__failed__": 0 }, "positive": { "negative": 2, "neutral": 2, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 14, "accuracy": 0.14, "wilson95": [ 0.08526343659939134, 0.22137188210963837 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 95, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.475, "brier": 0.253074285014304, "nll": 0.39543056071135163, "ece10": 0.017032617976585514, "calibration_n": 100, "confident_wrong": 2 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 238, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7933333333333333, "brier": 0.307218366234783, "nll": 0.6381570712293215, "ece10": 0.05050172358868716, "calibration_n": 300, "confident_wrong": 13 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 68, "accuracy": 0.68, "wilson95": [ 0.5833738138954762, 0.7633085267500092 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 68, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.68, "brier": 0.49288023578043777, "nll": 1.657923036307254, "ece10": 0.135954297106436, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.6610201656420145, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 2, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 5, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 2, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 3, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 3, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 5, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 5, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 2, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 1, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 35, "accuracy": 0.35, "wilson95": [ 0.26364248243097527, 0.44745556703112027 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35, "brier": 0.8887728874672531, "nll": 2.2596397476887575, "ece10": 0.31732476545847194, "calibration_n": 100, "confident_wrong": 7, "macro_f1": 0.3201069524345386, "confusion_matrix": { "billing": { "billing": 3, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 3, "account_access": 5, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 3, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 3, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 7, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 5, "account_access": 2, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 4, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 1, "onboarding": 2, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 6, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 3, "technical_outage": 1, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 1, "pricing": 1, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 4, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 1, "account_access": 0, "shipping": 2, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 2, "pricing": 1, "data_privacy": 1, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 12, "accuracy": 0.12, "wilson95": [ 0.06999406437019488, 0.19812099426711421 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 153, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.51, "brier": 0.6895584419599718, "nll": 1.7456964750594872, "ece10": 0.18998705510274566, "calibration_n": 300, "confident_wrong": 5 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.6285297737616997, "nll": 1.5289264072821882, "ece10": 0.2890296271823907, "calibration_n": 100, "confident_wrong": 9, "macro_f1": 0.5547161172161172, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 4, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 2, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 11, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 3, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 4, "car": 0, "visa_docs": 1, "loyalty": 4, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 5, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 4, "car": 2, "visa_docs": 2, "loyalty": 1, "disruption": 1, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.1558327506421664, "nll": 0.27606922419267843, "ece10": 0.06935368293283449, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8683996929589101, "confusion_matrix": { "business": { "business": 25, "science and technology": 0, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 9, "science and technology": 13, "sports": 1, "world": 2, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 77, "accuracy": 0.77, "wilson95": [ 0.6784561697712622, 0.8415673411969654 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 77, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.77, "brier": 0.33767798554277506, "nll": 0.8111589845503857, "ece10": 0.051783122429634815, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.7514550264550265, "confusion_matrix": { "refund not showing up": { "refund not showing up": 1, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 1, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 2, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 1, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 1, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 1, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 2, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 1, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 50, "accuracy": 0.5, "wilson95": [ 0.4038315303659956, 0.5961684696340044 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 50, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.7079397512334321, "nll": 1.546750672627011, "ece10": 0.19448262050410525, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.48667926276621926, "confusion_matrix": { "anger": { "anger": 12, "fear": 2, "joy": 0, "love": 0, "sadness": 3, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 1, "fear": 12, "joy": 1, "love": 0, "sadness": 2, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 9, "love": 2, "sadness": 3, "surprise": 2, "__failed__": 0 }, "love": { "anger": 2, "fear": 0, "joy": 3, "love": 5, "sadness": 7, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 3, "joy": 2, "love": 2, "sadness": 9, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 2, "fear": 1, "joy": 4, "love": 0, "sadness": 6, "surprise": 3, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2375, "accuracy": 0.8056309362279511, "wilson95": [ 0.7909524218774165, 0.8195139683872986 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2375, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8056309362279511, "brier": 0.2745147261921767, "nll": 0.6588548768625393, "ece10": 0.02597726689331599, "calibration_n": 2948, "confident_wrong": 78, "macro_f1": 0.8045504620912883, "confusion_matrix": { "alarm": { "alarm": 88, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 47, "calendar": 2, "cooking": 0, "datetime": 3, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 1, "audio": 0, "calendar": 360, "cooking": 0, "datetime": 8, "email": 0, "general": 3, "iot": 0, "lists": 2, "music": 0, "news": 4, "play": 0, "qa": 2, "recommendation": 17, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 69, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 94, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 4, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 1, "email": 232, "general": 11, "iot": 0, "lists": 6, "music": 0, "news": 0, "play": 0, "qa": 4, "recommendation": 2, "social": 5, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 13, "cooking": 7, "datetime": 2, "email": 0, "general": 85, "iot": 4, "lists": 0, "music": 1, "news": 1, "play": 3, "qa": 36, "recommendation": 22, "social": 1, "takeaway": 2, "transport": 5, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 7, "calendar": 0, "cooking": 9, "datetime": 0, "email": 0, "general": 4, "iot": 179, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 13, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 1, "audio": 1, "calendar": 7, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 1, "lists": 119, "music": 5, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 4, "iot": 0, "lists": 1, "music": 72, "news": 0, "play": 0, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 105, "play": 1, "qa": 10, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 1, "weather": 1, "__failed__": 0 }, "play": { "alarm": 0, "audio": 6, "calendar": 0, "cooking": 0, "datetime": 4, "email": 0, "general": 17, "iot": 1, "lists": 1, "music": 140, "news": 2, "play": 208, "qa": 4, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 3, "email": 0, "general": 12, "iot": 1, "lists": 0, "music": 3, "news": 8, "play": 1, "qa": 246, "recommendation": 6, "social": 0, "takeaway": 0, "transport": 3, "weather": 4, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 3, "play": 1, "qa": 3, "recommendation": 83, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 0, "email": 1, "general": 21, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 76, "takeaway": 2, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 52, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 6, "social": 0, "takeaway": 0, "transport": 115, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 145, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2133, "accuracy": 0.7235413839891451, "wilson95": [ 0.7071136166964754, 0.7393873280143427 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2133, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7235413839891451, "brier": 0.38823260806306087, "nll": 0.9954726332549508, "ece10": 0.058492985986663755, "calibration_n": 2948, "confident_wrong": 76, "macro_f1": 0.7204681338272033, "confusion_matrix": { "alarm": { "alarm": 86, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 2, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 38, "calendar": 1, "cooking": 0, "datetime": 5, "email": 0, "general": 12, "iot": 1, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 0, "audio": 0, "calendar": 336, "cooking": 0, "datetime": 12, "email": 2, "general": 7, "iot": 1, "lists": 3, "music": 0, "news": 2, "play": 0, "qa": 3, "recommendation": 25, "social": 1, "takeaway": 5, "transport": 2, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 66, "datetime": 0, "email": 0, "general": 0, "iot": 1, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 97, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 5, "cooking": 1, "datetime": 5, "email": 223, "general": 15, "iot": 1, "lists": 2, "music": 0, "news": 1, "play": 0, "qa": 8, "recommendation": 2, "social": 4, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 12, "cooking": 8, "datetime": 3, "email": 1, "general": 85, "iot": 3, "lists": 1, "music": 1, "news": 4, "play": 2, "qa": 34, "recommendation": 19, "social": 1, "takeaway": 0, "transport": 7, "weather": 5, "__failed__": 0 }, "iot": { "alarm": 3, "audio": 7, "calendar": 1, "cooking": 10, "datetime": 4, "email": 0, "general": 7, "iot": 161, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 15, "transport": 0, "weather": 2, "__failed__": 0 }, "lists": { "alarm": 1, "audio": 0, "calendar": 5, "cooking": 2, "datetime": 1, "email": 0, "general": 8, "iot": 2, "lists": 114, "music": 4, "news": 0, "play": 1, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 2, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 66, "news": 1, "play": 1, "qa": 4, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "news": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 0, "lists": 1, "music": 0, "news": 85, "play": 0, "qa": 18, "recommendation": 8, "social": 0, "takeaway": 0, "transport": 2, "weather": 2, "__failed__": 0 }, "play": { "alarm": 1, "audio": 24, "calendar": 1, "cooking": 4, "datetime": 10, "email": 0, "general": 35, "iot": 0, "lists": 5, "music": 169, "news": 3, "play": 114, "qa": 8, "recommendation": 5, "social": 0, "takeaway": 1, "transport": 3, "weather": 2, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 2, "calendar": 1, "cooking": 0, "datetime": 11, "email": 0, "general": 18, "iot": 2, "lists": 0, "music": 2, "news": 7, "play": 1, "qa": 229, "recommendation": 8, "social": 0, "takeaway": 0, "transport": 2, "weather": 4, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 3, "recommendation": 83, "social": 0, "takeaway": 1, "transport": 2, "weather": 1, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 0, "email": 2, "general": 27, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 4, "social": 62, "takeaway": 5, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 2, "datetime": 2, "email": 0, "general": 2, "iot": 2, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 45, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 2, "email": 0, "general": 1, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 6, "social": 0, "takeaway": 0, "transport": 110, "weather": 2, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 2, "email": 0, "general": 5, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 5, "social": 0, "takeaway": 0, "transport": 1, "weather": 133, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 813, "accuracy": 0.6930946291560103, "wilson95": [ 0.6661063653450134, 0.7188222897466832 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 813, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6930946291560103, "cluster_bootstrap95": [ 0.6607472077556619, 0.7246506153122114 ], "independent_source_groups": 47, "brier": 0.4226647812483118, "nll": 0.7256537643443217, "ece10": 0.021797805623465602, "calibration_n": 1173, "confident_wrong": 17, "macro_f1": 0.6615294270965061, "confusion_matrix": { "negative": { "negative": 108, "neutral": 53, "positive": 21, "__failed__": 0 }, "neutral": { "negative": 71, "neutral": 409, "positive": 110, "__failed__": 0 }, "positive": { "negative": 42, "neutral": 63, "positive": 296, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "julia": { "label": "Julia 144M", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 179, "accuracy": 0.895, "wilson95": [ 0.8448196800887696, 0.9302925120731899 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 179, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.895, "brier": 0.18595224788226397, "nll": 0.35702548192933997, "ece10": 0.04912369879480221, "calibration_n": 200, "confident_wrong": 5, "macro_f1": 0.8948281130634071, "confusion_matrix": { "world": { "world": 49, "sports": 0, "business": 0, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 7, "sports": 38, "business": 2, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 3, "sports": 0, "business": 44, "sci_tech": 4, "__failed__": 0 }, "sci_tech": { "world": 1, "sports": 0, "business": 3, "sci_tech": 48, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 182, "accuracy": 0.91, "wilson95": [ 0.8622342812328346, 0.9423125511378069 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 182, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.91, "brier": 0.10502482961504392, "nll": 0.1737727939268001, "ece10": 0.054388353909717996, "calibration_n": 200, "confident_wrong": 5, "macro_f1": 0.7892222761980361, "confusion_matrix": { "sadness": { "sadness": 54, "joy": 0, "love": 0, "anger": 3, "fear": 0, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 1, "joy": 75, "love": 3, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 0, "joy": 4, "love": 8, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 1, "joy": 0, "love": 0, "anger": 27, "fear": 0, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 0, "joy": 0, "love": 0, "anger": 2, "fear": 17, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 93, "accuracy": 0.465, "wilson95": [ 0.3971856415700402, 0.5341335312763685 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 93, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.465, "brier": 0.8804156269725276, "nll": 1.8222228937411997, "ece10": 0.40798798872438324, "calibration_n": 200, "confident_wrong": 65, "macro_f1": 0.46498662466561663, "confusion_matrix": { "false": { "false": 47, "true": 62, "__failed__": 0 }, "true": { "false": 45, "true": 46, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 76, "accuracy": 0.38, "wilson95": [ 0.31559005848673627, 0.44893281984380773 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 76, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.38, "brier": 1.1875607890785311, "nll": 3.7367463327042767, "ece10": 0.6049993404032229, "calibration_n": 200, "confident_wrong": 111, "macro_f1": 0.3140834163071136, "confusion_matrix": { "true": { "true": 69, "false": 8, "__failed__": 0 }, "false": { "true": 116, "false": 7, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 3, "accuracy": 0.25, "wilson95": [ 0.08894166839405471, 0.5323053349335657 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 3, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.25, "brier": 1.3896359552510578, "nll": 5.663962321640011, "ece10": 0.7087235758345147, "calibration_n": 12, "confident_wrong": 7, "macro_f1": 0.13, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "it": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "review": { "maintenance": 3, "procurement": 0, "documents": 0, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 4, "accuracy": 0.3333333333333333, "wilson95": [ 0.13812009109121315, 0.6093779111272004 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3333333333333333, "brier": 0.9937330697397876, "nll": 2.292101600737936, "ece10": 0.47123583829328586, "calibration_n": 12, "confident_wrong": 1, "macro_f1": 0.2333333333333333, "confusion_matrix": { "maintenance": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 6, "accuracy": 0.5, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 1.03513261436743, "nll": 2.9766866013593725, "ece10": 0.5599976176595656, "calibration_n": 12, "confident_wrong": 6, "macro_f1": 0.48636363636363633, "confusion_matrix": { "design": { "design": 1, "inspection": 2, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 1, "purchase": 1, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 2, "purchase": 0, "work": 1, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 5, "accuracy": 0.4166666666666667, "wilson95": [ 0.19326031365875665, 0.6804886874504503 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4166666666666667, "brier": 0.7365998430346599, "nll": 1.3250763494470215, "ece10": 0.3453708028060722, "calibration_n": 12, "confident_wrong": 2, "macro_f1": 0.42500000000000004, "confusion_matrix": { "design": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 2, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 2, "work": 0, "__failed__": 0 }, "work": { "design": 2, "inspection": 0, "purchase": 0, "work": 1, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 94, "accuracy": 0.94, "wilson95": [ 0.8752318455410406, 0.9722138760368118 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 94, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.94, "brier": 0.0858305675150916, "nll": 0.23014425748017062, "ece10": 0.025964249105087937, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.9399173646545361, "confusion_matrix": { "label_000": { "label_000": 22, "label_001": 2, "label_002": 1, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 23, "label_002": 1, "label_003": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 1, "label_002": 0, "label_003": 24, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 86, "accuracy": 0.86, "wilson95": [ 0.7786281178903616, 0.9147365634006086 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.25298841299993996, "nll": 0.9901523885297538, "ece10": 0.12537750112833054, "calibration_n": 100, "confident_wrong": 12, "macro_f1": 0.8556934538891329, "confusion_matrix": { "label_000": { "label_000": 16, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 1, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 16, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 16, "label_003": 1, "label_004": 0, "label_005": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 4, "label_003": 13, "label_004": 0, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 15, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 2, "label_002": 1, "label_003": 0, "label_004": 3, "label_005": 10, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "label_000": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_001": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 35, "accuracy": 0.7291666666666666, "wilson95": [ 0.5900303501775939, 0.8343404237254355 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7291666666666666, "brier": 0.4389980536855688, "nll": 1.2636661657829265, "ece10": 0.22119459734779356, "calibration_n": 48, "confident_wrong": 9 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 40, "accuracy": 0.36036036036036034, "wilson95": [ 0.2770990015212276, 0.4529636403300803 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.36036036036036034, "brier": 1.054420348957366, "nll": 3.3249370182102886, "ece10": 0.4654250721640907, "calibration_n": 111, "confident_wrong": 36 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 34, "accuracy": 0.4722222222222222, "wilson95": [ 0.36126510337834705, 0.5859932950992339 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4722222222222222, "cluster_bootstrap95": [ 0.3194444444444444, 0.6111111111111112 ], "independent_source_groups": 36, "brier": 0.940004793672472, "nll": 3.6830849559151955, "ece10": 0.4601354448173436, "calibration_n": 72, "confident_wrong": 22 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 12, "accuracy": 0.631578947368421, "wilson95": [ 0.41039533164803066, 0.8085048725486492 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.631578947368421, "brier": 0.5472484332536105, "nll": 0.9710342453009883, "ece10": 0.24998430786537937, "calibration_n": 19, "confident_wrong": 4, "macro_f1": 0.5521885521885521, "confusion_matrix": { "false": { "false": 10, "true": 1, "__failed__": 0 }, "true": { "false": 6, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.7343265195699397, "nll": 1.3949177587942647, "ece10": 0.3842767186110563, "calibration_n": 16, "confident_wrong": 2, "macro_f1": 0.4666666666666667, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 6, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 11, "accuracy": 0.5, "wilson95": [ 0.3072210627372502, 0.6927789372627497 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.9709726106355082, "nll": 2.74106791516103, "ece10": 0.4533045108441419, "calibration_n": 22, "confident_wrong": 10, "macro_f1": 0.39950372208436724, "confusion_matrix": { "false": { "false": 10, "true": 0, "__failed__": 0 }, "true": { "false": 11, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.5990868525560448, "nll": 2.6000083660774784, "ece10": 0.3365728800020947, "calibration_n": 16, "confident_wrong": 4, "macro_f1": 0.6904761904761905, "confusion_matrix": { "bug_report": { "bug_report": 4, "feature_request": 1, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 2, "praise": 1, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 2, "feature_request": 0, "praise": 0, "pricing_billing": 2, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 7, "accuracy": 0.3888888888888889, "wilson95": [ 0.2030524657216362, 0.613809583977621 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3888888888888889, "brier": 1.070480011566787, "nll": 3.570562211198071, "ece10": 0.5379729469663053, "calibration_n": 18, "confident_wrong": 7, "macro_f1": 0.32999999999999996, "confusion_matrix": { "0": { "0": 0, "1": 3, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 1, "1": 1, "2": 0, "3": 1, "4": 1, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 2, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 11, "accuracy": 0.5789473684210527, "wilson95": [ 0.3627590353425504, 0.7685810871754576 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5789473684210527, "brier": 0.7921492144333956, "nll": 2.710140649310382, "ece10": 0.41363856608727034, "calibration_n": 19, "confident_wrong": 5, "macro_f1": 0.553076923076923, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 2, "roads": 2, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 1, "roads": 1, "noise": 1, "permits": 1, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 1, "noise": 0, "permits": 3, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 2, "roads": 0, "noise": 0, "permits": 0, "utilities": 1, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 5, "accuracy": 0.29411764705882354, "wilson95": [ 0.1327998961029523, 0.5313311006653985 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.29411764705882354, "brier": 1.2442870383042202, "nll": 3.419440915817621, "ece10": 0.5979361561710889, "calibration_n": 17, "confident_wrong": 5, "macro_f1": 0.1994949494949495, "confusion_matrix": { "bug": { "bug": 2, "performance": 0, "security": 0, "style": 1, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 1, "performance": 0, "security": 0, "style": 1, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 1, "performance": 0, "security": 0, "style": 2, "refactor": 1, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 2, "refactor": 0, "nit": 1, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 1, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "nit": { "bug": 1, "performance": 0, "security": 0, "style": 2, "refactor": 0, "nit": 1, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 7, "accuracy": 0.4666666666666667, "wilson95": [ 0.24809535365737387, 0.698830199745016 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.8900864705026372, "nll": 3.127830376596061, "ece10": 0.38830313708971503, "calibration_n": 15, "confident_wrong": 4, "macro_f1": 0.43253968253968256, "confusion_matrix": { "feature": { "feature": 1, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 1, "refactor": 1, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 1, "refactor": 1, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 1, "docs": 1, "test": 0, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 0, "chore": 3, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.9692248390657904, "nll": 1.9383922568595207, "ece10": 0.4719889228457572, "calibration_n": 17, "confident_wrong": 7, "macro_f1": 0.43956043956043955, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 7, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.9444700030005115, "nll": 2.6624390461770857, "ece10": 0.5404279859070585, "calibration_n": 16, "confident_wrong": 7, "macro_f1": 0.5607843137254902, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 4, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 5, "accuracy": 0.2777777777777778, "wilson95": [ 0.12499753379553993, 0.5087265656029746 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2777777777777778, "brier": 1.2174894182160911, "nll": 5.094566889255398, "ece10": 0.5429643748280857, "calibration_n": 18, "confident_wrong": 4, "macro_f1": 0.2496969696969697, "confusion_matrix": { "harassment": { "harassment": 0, "threat": 3, "self_harm": 2, "spam": 0, "none": 0, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 1, "self_harm": 0, "spam": 1, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 1, "self_harm": 0, "spam": 3, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 4, "self_harm": 1, "spam": 1, "none": 0, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 9, "accuracy": 0.4090909090909091, "wilson95": [ 0.2325581619647187, 0.6126518152368488 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4090909090909091, "brier": 1.0988912885439641, "nll": 3.9378809482109385, "ece10": 0.5817709251862906, "calibration_n": 22, "confident_wrong": 7, "macro_f1": 0.35126050420168065, "confusion_matrix": { "news_report": { "news_report": 0, "opinion": 0, "press_release": 2, "analysis": 2, "feature": 1, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 1, "press_release": 3, "analysis": 0, "feature": 1, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 3, "analysis": 1, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 0, "opinion": 0, "press_release": 1, "analysis": 0, "feature": 3, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.8989600337876869, "nll": 3.308490262177506, "ece10": 0.4499080867693463, "calibration_n": 17, "confident_wrong": 7, "macro_f1": 0.5166666666666667, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 3, "termination": 0, "indemnification": 1, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 1, "confidentiality": 1, "termination": 1, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 1, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 2, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 1, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 12, "accuracy": 0.631578947368421, "wilson95": [ 0.41039533164803066, 0.8085048725486492 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.631578947368421, "brier": 0.5360046081174524, "nll": 1.3376944388506626, "ece10": 0.2963101926359169, "calibration_n": 19, "confident_wrong": 3, "macro_f1": 0.5466666666666666, "confusion_matrix": { "address_issue": { "address_issue": 2, "customs_hold": 1, "weather_delay": 0, "failed_attempt": 1, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 4, "weather_delay": 0, "failed_attempt": 1, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 3, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 1, "damaged": 3, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.8274447681496402, "nll": 2.1127012919420434, "ece10": 0.4354779305564428, "calibration_n": 17, "confident_wrong": 6, "macro_f1": 0.48484848484848486, "confusion_matrix": { "false": { "false": 2, "true": 7, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 7, "accuracy": 0.4666666666666667, "wilson95": [ 0.24809535365737387, 0.698830199745016 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.9655191125522758, "nll": 3.0194894976631845, "ece10": 0.513207582688022, "calibration_n": 15, "confident_wrong": 5, "macro_f1": 0.4, "confusion_matrix": { "invoice": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 3, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 1, "receipt": 0, "contract": 1, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 3, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 1, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 2, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 7, "accuracy": 0.4666666666666667, "wilson95": [ 0.24809535365737387, 0.698830199745016 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.9650280026516319, "nll": 3.4837933335562554, "ece10": 0.44795074750634173, "calibration_n": 15, "confident_wrong": 6, "macro_f1": 0.4133333333333333, "confusion_matrix": { "meeting_request": { "meeting_request": 1, "question": 1, "complaint": 1, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 1, "complaint": 2, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 2, "complaint": 0, "thanks": 1, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 2, "complaint": 0, "thanks": 1, "fyi": 0, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.5002493259795303, "nll": 0.9639919816588214, "ece10": 0.1822979131479692, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.5714285714285714, "confusion_matrix": { "meals": { "meals": 0, "travel": 1, "office_supplies": 0, "software": 0, "client_entertainment": 1, "__failed__": 0 }, "travel": { "meals": 0, "travel": 4, "office_supplies": 0, "software": 1, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 1, "office_supplies": 3, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 1, "office_supplies": 0, "software": 3, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 1, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 8, "accuracy": 0.4444444444444444, "wilson95": [ 0.24559518127005633, 0.6628358435795723 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4444444444444444, "brier": 0.8563137155213233, "nll": 1.7160564358444022, "ece10": 0.4373950339425494, "calibration_n": 18, "confident_wrong": 5, "macro_f1": 0.375, "confusion_matrix": { "false": { "false": 7, "true": 2, "__failed__": 0 }, "true": { "false": 8, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 3, "accuracy": 0.1875, "wilson95": [ 0.06591599071428142, 0.4300888096197414 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 3, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.1875, "brier": 1.4428892064704912, "nll": 5.508778858782735, "ece10": 0.7214395489014503, "calibration_n": 16, "confident_wrong": 9, "macro_f1": 0.1875, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 2, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 1, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 3, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 6, "1": 0, "2": 0, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 8, "accuracy": 0.4444444444444444, "wilson95": [ 0.24559518127005633, 0.6628358435795723 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4444444444444444, "brier": 1.0032407998748822, "nll": 2.320482632480096, "ece10": 0.47603936403766717, "calibration_n": 18, "confident_wrong": 6, "macro_f1": 0.4006238859180036, "confusion_matrix": { "0": { "0": 2, "1": 2, "2": 1, "__failed__": 0 }, "1": { "0": 2, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 2, "1": 3, "2": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 8, "accuracy": 0.4, "wilson95": [ 0.21880653237281705, 0.6134184992377469 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.9998977929202153, "nll": 2.5838010761492014, "ece10": 0.502370109958821, "calibration_n": 20, "confident_wrong": 5, "macro_f1": 0.37230769230769234, "confusion_matrix": { "cheating": { "cheating": 1, "harassment": 2, "boosting": 0, "smurfing": 1, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "boosting": { "cheating": 1, "harassment": 0, "boosting": 1, "smurfing": 1, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 3, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 4, "boosting": 0, "smurfing": 1, "other": 0, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 4, "accuracy": 0.19047619047619047, "wilson95": [ 0.07667565052761793, 0.40000564690204926 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.19047619047619047, "brier": 1.5367758922397508, "nll": 5.870121066923908, "ece10": 0.78558364221876, "calibration_n": 21, "confident_wrong": 13, "macro_f1": 0.06956521739130435, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 1, "none": 1, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 }, "none": { "spelling": 1, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.7046249888016656, "nll": 1.083716983045199, "ece10": 0.352320964425342, "calibration_n": 16, "confident_wrong": 2, "macro_f1": 0.4666666666666667, "confusion_matrix": { "false": { "false": 2, "true": 5, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 13, "accuracy": 0.5652173913043478, "wilson95": [ 0.3681137508501646, 0.743653646743258 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5652173913043478, "brier": 0.6456196294041266, "nll": 1.5565942335813645, "ece10": 0.27310530763672686, "calibration_n": 23, "confident_wrong": 3, "macro_f1": 0.49428571428571433, "confusion_matrix": { "plumbing": { "plumbing": 4, "electrical": 0, "hvac": 1, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 2, "hvac": 3, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 1, "electrical": 1, "hvac": 0, "handyman": 3, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 2, "handyman": 2, "appliance_repair": 0, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 4, "accuracy": 0.2857142857142857, "wilson95": [ 0.11721378644576919, 0.5464908432989318 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2857142857142857, "brier": 1.3201829055187826, "nll": 5.87944221114672, "ece10": 0.618820056220101, "calibration_n": 14, "confident_wrong": 7, "macro_f1": 0.18571428571428572, "confusion_matrix": { "0": { "0": 1, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 0, "2": 1, "3": 4, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 1, "3": 1, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 4, "accuracy": 0.25, "wilson95": [ 0.10182067491213048, 0.49498316535508774 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.25, "brier": 1.2773242236406928, "nll": 3.762301558970337, "ece10": 0.656283636866916, "calibration_n": 16, "confident_wrong": 5, "macro_f1": 0.13333333333333333, "confusion_matrix": { "0": { "0": 0, "1": 0, "2": 2, "3": 2, "__failed__": 0 }, "1": { "0": 0, "1": 0, "2": 2, "3": 3, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 0, "3": 2, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 5, "accuracy": 0.3125, "wilson95": [ 0.14164643854782039, 0.5559564416525933 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3125, "brier": 1.099491028573234, "nll": 3.100746262111409, "ece10": 0.4769743918453194, "calibration_n": 16, "confident_wrong": 3, "macro_f1": 0.225, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 0, "3": 2, "__failed__": 0 }, "1": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 }, "2": { "0": 1, "1": 1, "2": 0, "3": 2, "__failed__": 0 }, "3": { "0": 1, "1": 0, "2": 0, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.6582550802835812, "nll": 1.2105255788788425, "ece10": 0.3028219416838708, "calibration_n": 17, "confident_wrong": 3, "macro_f1": 0.5641025641025641, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 6, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.5950449984958406, "nll": 1.1827127762979868, "ece10": 0.29574696329534594, "calibration_n": 17, "confident_wrong": 2, "macro_f1": 0.5492063492063493, "confusion_matrix": { "politics": { "politics": 1, "business": 1, "technology": 1, "sports": 0, "culture": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.5945269076663242, "nll": 1.4780229915807364, "ece10": 0.2931421458691501, "calibration_n": 17, "confident_wrong": 3, "macro_f1": 0.651111111111111, "confusion_matrix": { "payments": { "payments": 1, "web": 0, "infra": 3, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 1, "infra": 3, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 1, "data": 2, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 1, "data": 0, "model": 1, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 5, "accuracy": 0.29411764705882354, "wilson95": [ 0.1327998961029523, 0.5313311006653985 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.29411764705882354, "brier": 1.062408779206019, "nll": 1.9249035653197324, "ece10": 0.6120862080636988, "calibration_n": 17, "confident_wrong": 4, "macro_f1": 0.27142857142857146, "confusion_matrix": { "false": { "false": 4, "true": 4, "__failed__": 0 }, "true": { "false": 8, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 7, "accuracy": 0.4375, "wilson95": [ 0.23098652405492354, 0.6682144360118811 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4375, "brier": 0.8405804855449233, "nll": 1.4277522134133036, "ece10": 0.46186172812383497, "calibration_n": 16, "confident_wrong": 2, "macro_f1": 0.43529411764705883, "confusion_matrix": { "false": { "false": 3, "true": 5, "__failed__": 0 }, "true": { "false": 4, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 5, "accuracy": 0.3125, "wilson95": [ 0.14164643854782039, 0.5559564416525933 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3125, "brier": 1.3589474935215653, "nll": 5.154842570600827, "ece10": 0.7123491282593928, "calibration_n": 16, "confident_wrong": 10, "macro_f1": 0.15873015873015872, "confusion_matrix": { "0": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 6, "2": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 7, "accuracy": 0.3888888888888889, "wilson95": [ 0.2030524657216362, 0.613809583977621 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3888888888888889, "brier": 0.9618822810100662, "nll": 4.811069776502166, "ece10": 0.42653401772589444, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.3106060606060606, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 2, "french": 0, "mexican": 2, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 2, "french": 0, "mexican": 0, "indian": 1, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 2, "indian": 0, "thai": 0, "middle_eastern": 1, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 7, "accuracy": 0.4117647058823529, "wilson95": [ 0.2161105679021481, 0.6399455735700021 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4117647058823529, "brier": 1.0768293526391992, "nll": 2.1272779770148458, "ece10": 0.5936272203857895, "calibration_n": 17, "confident_wrong": 8, "macro_f1": 0.39285714285714285, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 7, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 4, "accuracy": 0.2222222222222222, "wilson95": [ 0.09000928108601694, 0.45214584316212625 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2222222222222222, "brier": 1.5432632251933958, "nll": 7.630873772302469, "ece10": 0.7716997640064271, "calibration_n": 18, "confident_wrong": 14, "macro_f1": 0.13999999999999999, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 3, "wrong_item": 1, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 4, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 4, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 6, "accuracy": 0.3333333333333333, "wilson95": [ 0.1627877475083369, 0.5625053270405489 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3333333333333333, "brier": 1.1099441066553817, "nll": 4.475606530946729, "ece10": 0.5423141223722934, "calibration_n": 18, "confident_wrong": 6, "macro_f1": 0.30828282828282827, "confusion_matrix": { "0": { "0": 1, "1": 1, "2": 0, "3": 1, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 1, "4": 2, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 1, "__failed__": 0 }, "4": { "0": 1, "1": 1, "2": 0, "3": 0, "4": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.6470517775822955, "nll": 1.274566059035829, "ece10": 0.3999052953586021, "calibration_n": 16, "confident_wrong": 2, "macro_f1": 0.6, "confusion_matrix": { "false": { "false": 7, "true": 3, "__failed__": 0 }, "true": { "false": 3, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 8, "accuracy": 0.5333333333333333, "wilson95": [ 0.30116980025498397, 0.7519046463426261 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5333333333333333, "brier": 0.9125966413745902, "nll": 3.009763686934387, "ece10": 0.450200593030079, "calibration_n": 15, "confident_wrong": 7, "macro_f1": 0.34782608695652173, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 7, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.6515743004691739, "nll": 1.6286826056067938, "ece10": 0.29072410151681793, "calibration_n": 17, "confident_wrong": 3, "macro_f1": 0.5976190476190475, "confusion_matrix": { "billing": { "billing": 2, "tech": 2, "sales": 1, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 1, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 2, "sales": 0, "account": 1, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 1, "sales": 0, "account": 0, "other": 1, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.8712646275657268, "nll": 1.7220039830881333, "ece10": 0.46346608699323544, "calibration_n": 17, "confident_wrong": 4, "macro_f1": 0.32, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 9, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 7, "accuracy": 0.3684210526315789, "wilson95": [ 0.19149512745135072, 0.5896046683519693 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3684210526315789, "brier": 1.2329645212790779, "nll": 6.9095506873058765, "ece10": 0.5949040960400659, "calibration_n": 19, "confident_wrong": 11, "macro_f1": 0.1346153846153846, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 2, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.511074277927859, "nll": 0.9471967067651894, "ece10": 0.21312073366284157, "calibration_n": 17, "confident_wrong": 2, "macro_f1": 0.6136363636363636, "confusion_matrix": { "false": { "false": 8, "true": 3, "__failed__": 0 }, "true": { "false": 3, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 8, "accuracy": 0.4444444444444444, "wilson95": [ 0.24559518127005633, 0.6628358435795723 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4444444444444444, "brier": 0.8247430539961803, "nll": 1.580553285645212, "ece10": 0.404712033659121, "calibration_n": 18, "confident_wrong": 3, "macro_f1": 0.41558441558441556, "confusion_matrix": { "false": { "false": 2, "true": 6, "__failed__": 0 }, "true": { "false": 4, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 6, "accuracy": 0.3, "wilson95": [ 0.1454772448676043, 0.5189728183535235 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3, "brier": 1.07584830556423, "nll": 3.577893115822936, "ece10": 0.5392431562043216, "calibration_n": 20, "confident_wrong": 7, "macro_f1": 0.225, "confusion_matrix": { "0": { "0": 4, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 2, "1": 1, "2": 0, "3": 1, "__failed__": 0 }, "3": { "0": 3, "1": 0, "2": 0, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 9, "accuracy": 0.45, "wilson95": [ 0.2581978582556243, 0.6579146575496577 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.9580439710578015, "nll": 3.314765211174975, "ece10": 0.4783560290535368, "calibration_n": 20, "confident_wrong": 7, "macro_f1": 0.42666666666666664, "confusion_matrix": { "device_control": { "device_control": 1, "information_request": 0, "reminder": 3, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 1, "reminder": 3, "music": 1, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 2, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 2, "music": 0, "small_talk": 2, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 10, "accuracy": 0.5, "wilson95": [ 0.2992980081982123, 0.7007019918017877 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.8650488736786917, "nll": 1.8436064305017048, "ece10": 0.4161707304869502, "calibration_n": 20, "confident_wrong": 6, "macro_f1": 0.45054945054945056, "confusion_matrix": { "false": { "false": 2, "true": 7, "__failed__": 0 }, "true": { "false": 3, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 11, "accuracy": 0.5, "wilson95": [ 0.3072210627372502, 0.6927789372627497 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.8133022164614073, "nll": 2.063064019117924, "ece10": 0.3309919638291141, "calibration_n": 22, "confident_wrong": 6, "macro_f1": 0.48276942355889724, "confusion_matrix": { "0": { "0": 1, "1": 3, "2": 0, "3": 1, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 2, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 3, "2": 0, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 326, "accuracy": 0.3622222222222222, "wilson95": [ 0.3314677237605492, 0.39414787222728787 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 436, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4063373718546132, "cluster_bootstrap95": [ 0.3086895993806083, 0.40641196977090266 ], "independent_source_groups": 596, "brier": 0.9600976922732104, "nll": 2.7071289801956215, "ece10": 0.4308701424938488, "calibration_n": 1073, "confident_wrong": 299 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 195, "accuracy": 0.3397212543554007, "wilson95": [ 0.3021559501427648, 0.37941760746329173 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 459, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49038461538461536, "brier": 0.8117239484939882, "nll": 2.4891378164705165, "ece10": 0.35477998621846674, "calibration_n": 936, "confident_wrong": 204 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 159, "accuracy": 0.22714285714285715, "wilson95": [ 0.197642762070111, 0.259621377157988 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 361, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3318014705882353, "cluster_bootstrap95": [ 0.19568267125430092, 0.2592671394799054 ], "independent_source_groups": 650, "brier": 1.069407326022463, "nll": 3.1634400856334333, "ece10": 0.4595313563055117, "calibration_n": 1088, "confident_wrong": 303 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 107, "accuracy": 0.2675, "wilson95": [ 0.22648238411831478, 0.31294083274713314 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1463, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7315, "brier": 0.4932098217790163, "nll": 1.92761939698549, "ece10": 0.22731428092074338, "calibration_n": 2000, "confident_wrong": 414 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 18, "accuracy": 0.6428571428571429, "wilson95": [ 0.4583030651442781, 0.7929417049226233 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6428571428571429, "brier": 0.5733475376351213, "nll": 1.4445885258282918, "ece10": 0.2504322358497524, "calibration_n": 28, "confident_wrong": 2, "macro_f1": 0.613458110516934, "confusion_matrix": { "design": { "design": 2, "inspection": 3, "purchase": 2, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 6, "purchase": 1, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 2, "inspection": 2, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 21, "accuracy": 0.75, "wilson95": [ 0.5664432968100104, 0.8732350508070668 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.45032117826696894, "nll": 1.0791340618180647, "ece10": 0.1691742416305628, "calibration_n": 28, "confident_wrong": 1, "macro_f1": 0.7480852480852481, "confusion_matrix": { "work": { "work": 6, "purchase": 0, "inspection": 0, "design": 1, "__failed__": 0 }, "purchase": { "work": 1, "purchase": 6, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 1, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 4, "purchase": 0, "inspection": 0, "design": 3, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 14, "accuracy": 0.5, "wilson95": [ 0.32633127457293465, 0.6736687254270654 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.7720846264712314, "nll": 2.127909185644324, "ece10": 0.3977803024038714, "calibration_n": 28, "confident_wrong": 7, "macro_f1": 0.4236111111111111, "confusion_matrix": { "design": { "design": 0, "inspection": 6, "purchase": 1, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 6, "purchase": 1, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 1, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 4, "purchase": 1, "work": 2, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 19, "accuracy": 0.6785714285714286, "wilson95": [ 0.49338843180857617, 0.8206675307750504 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6785714285714286, "brier": 0.5280447934054877, "nll": 1.2979877666694504, "ece10": 0.22464968525259332, "calibration_n": 28, "confident_wrong": 3, "macro_f1": 0.5823529411764705, "confusion_matrix": { "work": { "work": 6, "purchase": 0, "inspection": 1, "design": 0, "__failed__": 0 }, "purchase": { "work": 1, "purchase": 6, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 3, "purchase": 2, "inspection": 2, "design": 0, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 24, "accuracy": 0.6, "wilson95": [ 0.44595893660346186, 0.7365167431570808 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 24, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.6017797952018928, "nll": 1.6841360438878437, "ece10": 0.2410903202441536, "calibration_n": 40, "confident_wrong": 3, "macro_f1": 0.5404761904761906, "confusion_matrix": { "review": { "review": 5, "it": 0, "documents": 0, "procurement": 1, "maintenance": 2, "__failed__": 0 }, "it": { "review": 7, "it": 0, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 1, "it": 0, "documents": 5, "procurement": 2, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 2, "maintenance": 6, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 19, "accuracy": 0.475, "wilson95": [ 0.329354701126789, 0.6250263789330753 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.475, "brier": 0.8089359379027595, "nll": 2.008626355897781, "ece10": 0.3312541325295799, "calibration_n": 40, "confident_wrong": 9, "macro_f1": 0.4133333333333333, "confusion_matrix": { "maintenance": { "maintenance": 7, "procurement": 0, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "procurement": { "maintenance": 2, "procurement": 6, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 3, "procurement": 1, "documents": 3, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 3, "procurement": 1, "documents": 2, "it": 0, "review": 2, "__failed__": 0 }, "review": { "maintenance": 3, "procurement": 2, "documents": 0, "it": 0, "review": 3, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 17, "accuracy": 0.425, "wilson95": [ 0.28509392559950375, 0.5780493145800892 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.425, "brier": 0.9265940259376813, "nll": 2.6457103299418123, "ece10": 0.4259919186026583, "calibration_n": 40, "confident_wrong": 9, "macro_f1": 0.403921568627451, "confusion_matrix": { "review": { "review": 4, "it": 0, "documents": 0, "procurement": 1, "maintenance": 3, "__failed__": 0 }, "it": { "review": 4, "it": 1, "documents": 2, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 4, "procurement": 3, "maintenance": 1, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 2, "procurement": 5, "maintenance": 1, "__failed__": 0 }, "maintenance": { "review": 2, "it": 0, "documents": 0, "procurement": 3, "maintenance": 3, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 11, "accuracy": 0.275, "wilson95": [ 0.16108016247667337, 0.42834955806210573 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.275, "brier": 1.1629001635151475, "nll": 3.305407642740387, "ece10": 0.5774645546083825, "calibration_n": 40, "confident_wrong": 14, "macro_f1": 0.24333333333333335, "confusion_matrix": { "maintenance": { "maintenance": 3, "procurement": 5, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 3, "procurement": 3, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 2, "procurement": 2, "documents": 4, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 2, "procurement": 1, "documents": 2, "it": 0, "review": 3, "__failed__": 0 }, "review": { "maintenance": 6, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 376, "accuracy": 0.94, "wilson95": [ 0.9122771510995438, 0.9593520298862752 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 376, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.94, "brier": 0.08742846004482926, "nll": 0.1445193897689151, "ece10": 0.03413792197329445, "calibration_n": 400, "confident_wrong": 6, "macro_f1": 0.8913793978275777, "confusion_matrix": { "sadness": { "sadness": 117, "joy": 0, "love": 1, "anger": 1, "fear": 0, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 0, "joy": 118, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 0, "joy": 10, "love": 27, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 2, "joy": 0, "love": 0, "anger": 62, "fear": 0, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 1, "joy": 0, "love": 0, "anger": 3, "fear": 44, "surprise": 4, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 0, "love": 0, "anger": 0, "fear": 2, "surprise": 8, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 198, "accuracy": 0.66, "wilson95": [ 0.604674453304638, 0.7112797959035073 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 198, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.66, "brier": 0.4840479275569306, "nll": 1.3846036736133256, "ece10": 0.17118750710020192, "calibration_n": 300, "confident_wrong": 17 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 301, "accuracy": 0.7525, "wilson95": [ 0.7079392288053334, 0.7922570625558467 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 301, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7525, "brier": 0.37387179557202666, "nll": 0.7508789970089221, "ece10": 0.1341560013647156, "calibration_n": 400, "confident_wrong": 41, "macro_f1": 0.6114561563594619, "confusion_matrix": { "false": { "false": 271, "true": 38, "__failed__": 0 }, "true": { "false": 61, "true": 30, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 344, "accuracy": 0.86, "wilson95": [ 0.8225607694538968, 0.8905903786254097 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 344, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.2418126183404284, "nll": 0.45424085412136067, "ece10": 0.05825745508418519, "calibration_n": 400, "confident_wrong": 14, "macro_f1": 0.8548352297116473, "confusion_matrix": { "world": { "world": 96, "sports": 1, "business": 5, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 16, "sports": 107, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 7, "sports": 0, "business": 64, "sci_tech": 1, "__failed__": 0 }, "sci_tech": { "world": 6, "sports": 0, "business": 19, "sci_tech": 77, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 150, "accuracy": 0.375, "wilson95": [ 0.328957079463163, 0.4234209941204111 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 150, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.375, "brier": 1.1585100995448314, "nll": 3.636219026953374, "ece10": 0.5821086559427466, "calibration_n": 400, "confident_wrong": 217, "macro_f1": 0.30118798043326345, "confusion_matrix": { "true": { "true": 140, "false": 13, "__failed__": 0 }, "false": { "true": 237, "false": 10, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 156, "accuracy": 0.52, "wilson95": [ 0.4635710820137406, 0.5759231991372775 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 156, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.7336862233811695, "nll": 2.037246636791172, "ece10": 0.27486246493249744, "calibration_n": 300, "confident_wrong": 45 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 180, "accuracy": 0.45, "wilson95": [ 0.40195222801404185, 0.4989990014193878 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 180, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.9212973121494319, "nll": 1.9511396285687197, "ece10": 0.43306104849580074, "calibration_n": 400, "confident_wrong": 139, "macro_f1": 0.4449910441736673, "confusion_matrix": { "false": { "false": 71, "true": 117, "__failed__": 0 }, "true": { "false": 103, "true": 109, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 215, "accuracy": 0.5375, "wilson95": [ 0.4885138311983594, 0.5857727467265683 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 215, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5375, "brier": 0.6950769894846915, "nll": 1.2422589027978523, "ece10": 0.29248384918193193, "calibration_n": 400, "confident_wrong": 76, "macro_f1": 0.5364541246186709, "confusion_matrix": { "false": { "false": 117, "true": 83, "__failed__": 0 }, "true": { "false": 102, "true": 98, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 49, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.49, "brier": 0.8052577389802792, "nll": 1.6623277213215764, "ece10": 0.36536567986882607, "calibration_n": 100, "confident_wrong": 27, "macro_f1": 0.4498975299320462, "confusion_matrix": { "yes": { "yes": 11, "no": 43, "__failed__": 0 }, "no": { "yes": 8, "no": 38, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 43, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43, "brier": 0.8557220886466478, "nll": 2.653529414367501, "ece10": 0.3577465384047273, "calibration_n": 100, "confident_wrong": 19, "macro_f1": 0.4088539524201289, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 1, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 2, "dispute_charge": 2, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 3, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "mortgage": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 7, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 1, "transfer": 1, "card_lost": 1, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "statement": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 2, "statement": 3, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 6, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 2, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 4, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 2, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 5, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 1, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 3, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 1, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 1, "limit_increase": 0, "other": 1, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 3, "close_account": 0, "new_account": 0, "limit_increase": 3, "other": 0, "__failed__": 0 }, "other": { "balance": 1, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 1, "fraud_alert": 1, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 1, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 36, "accuracy": 0.36, "wilson95": [ 0.2727122040140433, 0.4576459754839126 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 126, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.63, "brier": 0.5854981631835501, "nll": 1.1520843878247746, "ece10": 0.2502921687353675, "calibration_n": 200, "confident_wrong": 35 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 20, "accuracy": 0.2, "wilson95": [ 0.1333669333310325, 0.2888291655931589 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.9872588381416213, "nll": 2.726492680936188, "ece10": 0.4573938539965132, "calibration_n": 200, "confident_wrong": 60 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 66, "accuracy": 0.66, "wilson95": [ 0.5627772885472462, 0.7453847920265184 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 66, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.66, "brier": 0.5306942539232756, "nll": 1.4616208106360205, "ece10": 0.19884885406043265, "calibration_n": 100, "confident_wrong": 9, "macro_f1": 0.6249624994925507, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 6, "receipt": 7, "contract": 3, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 5, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "bank_statement": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 14, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 1, "receipt": 0, "contract": 3, "resume": 1, "bank_statement": 0, "clinical_note": 5, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 3, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 1, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 7, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 2, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 2, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 4, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 1, "receipt": 0, "contract": 2, "resume": 2, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 2, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 4, "accuracy": 0.04, "wilson95": [ 0.01566330399154762, 0.0983707143588792 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 173, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4325, "brier": 0.9351006265665726, "nll": 2.50981954834337, "ece10": 0.42010416323431365, "calibration_n": 400, "confident_wrong": 113 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 47, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47, "brier": 0.7616652689252849, "nll": 2.1872353400428874, "ece10": 0.2479382621072252, "calibration_n": 100, "confident_wrong": 11, "macro_f1": 0.4252104377104377, "confusion_matrix": { "politics": { "politics": 0, "business": 1, "technology": 2, "science": 3, "health": 4, "sports": 0, "entertainment": 1, "world": 2, "culture": 0, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 7, "science": 2, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 1, "technology": 1, "science": 7, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 2, "technology": 0, "science": 2, "health": 1, "sports": 3, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 0, "business": 0, "technology": 2, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 2, "weather": 0, "__failed__": 0 }, "world": { "politics": 0, "business": 2, "technology": 1, "science": 2, "health": 0, "sports": 0, "entertainment": 0, "world": 4, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 1, "science": 4, "health": 2, "sports": 0, "entertainment": 0, "world": 3, "culture": 0, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 2, "health": 0, "sports": 0, "entertainment": 0, "world": 4, "culture": 0, "weather": 2, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 34, "accuracy": 0.34, "wilson95": [ 0.25461520797348164, 0.43722271145275377 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.34, "brier": 0.9214307774831823, "nll": 2.654236066936399, "ece10": 0.28158641791881084, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.24409195030587222, "confusion_matrix": { "biology": { "biology": 4, "medicine": 2, "chemistry": 0, "physics": 2, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 0, "medicine": 7, "chemistry": 0, "physics": 1, "computer_science": 1, "mathematics": 0, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 1, "chemistry": 4, "physics": 1, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 14, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 1, "medicine": 1, "chemistry": 3, "physics": 7, "computer_science": 2, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 1, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 1, "chemistry": 2, "physics": 2, "computer_science": 1, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 1, "medicine": 3, "chemistry": 1, "physics": 4, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 1, "medicine": 2, "chemistry": 0, "physics": 4, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 2, "medicine": 1, "chemistry": 0, "physics": 1, "computer_science": 2, "mathematics": 0, "earth_science": 1, "engineering": 0, "economics": 1, "other": 0, "__failed__": 0 }, "other": { "biology": 0, "medicine": 2, "chemistry": 2, "physics": 8, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.19, "brier": 1.0340938816861625, "nll": 2.7859595910767325, "ece10": 0.4661520386421354, "calibration_n": 100, "confident_wrong": 33 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 53, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.265, "brier": 0.8486963072300161, "nll": 2.0133118375452153, "ece10": 0.3313096342241364, "calibration_n": 100, "confident_wrong": 18 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 54, "accuracy": 0.54, "wilson95": [ 0.44264860323368227, 0.6343919169097589 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 54, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.54, "brier": 0.7275482251267735, "nll": 1.434382913405515, "ece10": 0.2813734789994702, "calibration_n": 100, "confident_wrong": 15, "macro_f1": 0.5175815296230492, "confusion_matrix": { "negative": { "negative": 27, "neutral": 6, "positive": 3, "__failed__": 0 }, "neutral": { "negative": 17, "neutral": 8, "positive": 8, "__failed__": 0 }, "positive": { "negative": 6, "neutral": 6, "positive": 19, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.345, "brier": 0.4831544545217503, "nll": 0.9689366137243364, "ece10": 0.20350399676108166, "calibration_n": 100, "confident_wrong": 12 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 10, "accuracy": 0.1, "wilson95": [ 0.0552291370606751, 0.17436566150491345 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 141, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47, "brier": 0.6942375693695982, "nll": 1.3733633843719992, "ece10": 0.25796711574086206, "calibration_n": 300, "confident_wrong": 24 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "order_status": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "refund_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 6 }, "shipping_delay": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "pricing_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "speak_to_human": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 7 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 9 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 7 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "promo_code": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "partner_inquiry": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 39, "accuracy": 0.39, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.39, "brier": 0.9345420038016055, "nll": 3.103086219848129, "ece10": 0.37109329027391713, "calibration_n": 100, "confident_wrong": 21, "macro_f1": 0.3695616390982524, "confusion_matrix": { "billing": { "billing": 4, "account_access": 0, "shipping": 1, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 4, "shipping": 1, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 5, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 2, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 0, "account_access": 1, "shipping": 1, "returns": 0, "technical_outage": 2, "onboarding": 0, "integrations": 1, "security": 2, "compliance": 0, "pricing": 2, "data_privacy": 0, "other": 2, "__failed__": 0 }, "onboarding": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 1, "integrations": 1, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 1, "shipping": 1, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 2, "compliance": 1, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 5, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 2, "pricing": 3, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 5, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 0, "shipping": 1, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 4, "other": 0, "__failed__": 0 }, "other": { "billing": 1, "account_access": 0, "shipping": 2, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 3, "data_privacy": 2, "other": 1, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 111, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.37, "brier": 0.9607455537425117, "nll": 2.4826204589245657, "ece10": 0.4067814234413894, "calibration_n": 300, "confident_wrong": 72 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 24, "accuracy": 0.24, "wilson95": [ 0.16691325556948772, 0.33232336349814473 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 24, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.24, "brier": 1.1502722802084806, "nll": 3.3862316797560394, "ece10": 0.46789673178511726, "calibration_n": 100, "confident_wrong": 21, "macro_f1": 0.2194678615731247, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 0, "cancel_trip": 1, "seat": 0, "baggage": 1, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 3, "change_flight": 2, "cancel_trip": 1, "seat": 0, "baggage": 1, "refund": 1, "hotel": 1, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 2, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 1, "seat": 4, "baggage": 0, "refund": 1, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 1, "change_flight": 0, "cancel_trip": 1, "seat": 0, "baggage": 7, "refund": 3, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 1, "cancel_trip": 1, "seat": 0, "baggage": 1, "refund": 3, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 2, "refund": 2, "hotel": 3, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 1, "seat": 0, "baggage": 1, "refund": 1, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 2, "change_flight": 1, "cancel_trip": 1, "seat": 0, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 1, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 1, "hotel": 3, "car": 2, "visa_docs": 1, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 2, "seat": 0, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 2, "other": 0, "__failed__": 0 }, "other": { "new_booking": 2, "change_flight": 0, "cancel_trip": 1, "seat": 0, "baggage": 1, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 0, "loyalty": 1, "disruption": 4, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 95, "accuracy": 0.95, "wilson95": [ 0.8882495307680808, 0.9784563208456319 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 95, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.11473578262455372, "nll": 0.24540580887400854, "ece10": 0.06957895570690743, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.9499098231764065, "confusion_matrix": { "business": { "business": 22, "science and technology": 2, "sports": 0, "world": 1, "__failed__": 0 }, "science and technology": { "business": 0, "science and technology": 24, "sports": 1, "world": 0, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 1, "sports": 24, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "refund not showing up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "activating a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.1825362232961633, "nll": 0.40025392656972736, "ece10": 0.0857449795986431, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.8781626458718653, "confusion_matrix": { "anger": { "anger": 16, "fear": 0, "joy": 0, "love": 0, "sadness": 1, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 0, "fear": 16, "joy": 0, "love": 0, "sadness": 0, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 16, "love": 1, "sadness": 0, "surprise": 0, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 4, "love": 13, "sadness": 0, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 1, "fear": 0, "joy": 0, "love": 0, "sadness": 15, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 0, "fear": 2, "joy": 0, "love": 0, "sadness": 2, "surprise": 12, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1256, "accuracy": 0.4260515603799186, "wilson95": [ 0.40830859347268206, 0.4439869968961216 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1256, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4260515603799186, "brier": 0.8796929092410544, "nll": 2.9808500483590747, "ece10": 0.34554561275866086, "calibration_n": 2948, "confident_wrong": 444, "macro_f1": 0.4456670741659139, "confusion_matrix": { "alarm": { "alarm": 82, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 7, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 7, "audio": 29, "calendar": 1, "cooking": 1, "datetime": 0, "email": 0, "general": 14, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 8, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 5, "audio": 0, "calendar": 141, "cooking": 6, "datetime": 158, "email": 0, "general": 23, "iot": 1, "lists": 2, "music": 0, "news": 5, "play": 1, "qa": 0, "recommendation": 35, "social": 1, "takeaway": 20, "transport": 1, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 58, "datetime": 3, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 9, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 1, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 82, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 8, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 3, "transport": 2, "weather": 0, "__failed__": 0 }, "email": { "alarm": 4, "audio": 0, "calendar": 26, "cooking": 0, "datetime": 13, "email": 47, "general": 85, "iot": 0, "lists": 4, "music": 0, "news": 11, "play": 1, "qa": 0, "recommendation": 1, "social": 53, "takeaway": 20, "transport": 2, "weather": 2, "__failed__": 0 }, "general": { "alarm": 12, "audio": 2, "calendar": 17, "cooking": 9, "datetime": 16, "email": 1, "general": 62, "iot": 3, "lists": 2, "music": 2, "news": 7, "play": 3, "qa": 1, "recommendation": 4, "social": 0, "takeaway": 38, "transport": 6, "weather": 2, "__failed__": 0 }, "iot": { "alarm": 121, "audio": 11, "calendar": 5, "cooking": 23, "datetime": 1, "email": 0, "general": 10, "iot": 11, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 31, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 5, "audio": 0, "calendar": 6, "cooking": 4, "datetime": 3, "email": 0, "general": 7, "iot": 1, "lists": 86, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 27, "transport": 1, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 15, "iot": 0, "lists": 4, "music": 41, "news": 4, "play": 10, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 4, "transport": 2, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 0, "lists": 0, "music": 0, "news": 110, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 2, "weather": 0, "__failed__": 0 }, "play": { "alarm": 14, "audio": 2, "calendar": 0, "cooking": 0, "datetime": 3, "email": 0, "general": 105, "iot": 1, "lists": 5, "music": 37, "news": 7, "play": 191, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 19, "transport": 1, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 17, "audio": 0, "calendar": 34, "cooking": 6, "datetime": 18, "email": 0, "general": 70, "iot": 13, "lists": 2, "music": 0, "news": 26, "play": 6, "qa": 0, "recommendation": 10, "social": 0, "takeaway": 60, "transport": 21, "weather": 4, "__failed__": 0 }, "recommendation": { "alarm": 3, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 10, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 0, "news": 11, "play": 4, "qa": 0, "recommendation": 35, "social": 0, "takeaway": 18, "transport": 5, "weather": 0, "__failed__": 0 }, "social": { "alarm": 2, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 0, "general": 31, "iot": 1, "lists": 1, "music": 0, "news": 5, "play": 0, "qa": 0, "recommendation": 1, "social": 51, "takeaway": 9, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 2, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 51, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 3, "audio": 0, "calendar": 6, "cooking": 0, "datetime": 13, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 16, "transport": 80, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 7, "audio": 2, "calendar": 3, "cooking": 4, "datetime": 10, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 21, "transport": 3, "weather": 99, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1174, "accuracy": 0.39823609226594303, "wilson95": [ 0.38070826254437656, 0.416028788421867 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1174, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.39823609226594303, "brier": 0.92000928733747, "nll": 3.117384788163091, "ece10": 0.3793865308575708, "calibration_n": 2948, "confident_wrong": 489, "macro_f1": 0.3916350082681237, "confusion_matrix": { "alarm": { "alarm": 83, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 5, "email": 0, "general": 3, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 19, "audio": 15, "calendar": 1, "cooking": 0, "datetime": 2, "email": 0, "general": 13, "iot": 0, "lists": 1, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 8, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 6, "audio": 0, "calendar": 129, "cooking": 4, "datetime": 131, "email": 0, "general": 43, "iot": 1, "lists": 13, "music": 0, "news": 16, "play": 1, "qa": 1, "recommendation": 16, "social": 2, "takeaway": 34, "transport": 2, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 57, "datetime": 4, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 9, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 2, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 89, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 2, "transport": 3, "weather": 1, "__failed__": 0 }, "email": { "alarm": 4, "audio": 0, "calendar": 19, "cooking": 0, "datetime": 11, "email": 88, "general": 40, "iot": 0, "lists": 1, "music": 0, "news": 28, "play": 0, "qa": 0, "recommendation": 0, "social": 72, "takeaway": 6, "transport": 0, "weather": 0, "__failed__": 0 }, "general": { "alarm": 11, "audio": 1, "calendar": 13, "cooking": 7, "datetime": 20, "email": 1, "general": 66, "iot": 2, "lists": 2, "music": 1, "news": 11, "play": 5, "qa": 1, "recommendation": 4, "social": 0, "takeaway": 33, "transport": 7, "weather": 2, "__failed__": 0 }, "iot": { "alarm": 83, "audio": 38, "calendar": 6, "cooking": 20, "datetime": 1, "email": 0, "general": 15, "iot": 6, "lists": 2, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 42, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 5, "audio": 0, "calendar": 14, "cooking": 8, "datetime": 1, "email": 0, "general": 13, "iot": 0, "lists": 73, "music": 1, "news": 2, "play": 3, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 22, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 3, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 2, "email": 0, "general": 25, "iot": 0, "lists": 7, "music": 19, "news": 2, "play": 13, "qa": 0, "recommendation": 1, "social": 1, "takeaway": 4, "transport": 2, "weather": 0, "__failed__": 0 }, "news": { "alarm": 2, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 4, "email": 0, "general": 10, "iot": 0, "lists": 2, "music": 0, "news": 98, "play": 2, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 3, "transport": 1, "weather": 0, "__failed__": 0 }, "play": { "alarm": 20, "audio": 4, "calendar": 4, "cooking": 0, "datetime": 5, "email": 2, "general": 103, "iot": 0, "lists": 0, "music": 16, "news": 10, "play": 199, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 21, "transport": 1, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 19, "audio": 1, "calendar": 24, "cooking": 10, "datetime": 28, "email": 0, "general": 66, "iot": 13, "lists": 4, "music": 1, "news": 33, "play": 8, "qa": 1, "recommendation": 10, "social": 0, "takeaway": 48, "transport": 16, "weather": 5, "__failed__": 0 }, "recommendation": { "alarm": 4, "audio": 0, "calendar": 6, "cooking": 3, "datetime": 13, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 8, "play": 5, "qa": 0, "recommendation": 29, "social": 0, "takeaway": 15, "transport": 8, "weather": 0, "__failed__": 0 }, "social": { "alarm": 3, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 3, "email": 0, "general": 35, "iot": 0, "lists": 1, "music": 0, "news": 7, "play": 1, "qa": 0, "recommendation": 0, "social": 44, "takeaway": 7, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 2, "datetime": 1, "email": 0, "general": 2, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 51, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 1, "audio": 0, "calendar": 6, "cooking": 1, "datetime": 20, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 23, "transport": 69, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 5, "audio": 0, "calendar": 16, "cooking": 9, "datetime": 14, "email": 0, "general": 9, "iot": 1, "lists": 0, "music": 0, "news": 21, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 15, "transport": 4, "weather": 58, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 541, "accuracy": 0.46121057118499575, "wilson95": [ 0.4328563632465923, 0.48981801288289034 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 541, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.46121057118499575, "cluster_bootstrap95": [ 0.4330561472267189, 0.48967865036187885 ], "independent_source_groups": 47, "brier": 0.8415490782999672, "nll": 1.8809944432611732, "ece10": 0.33373781254266044, "calibration_n": 1173, "confident_wrong": 254, "macro_f1": 0.41283182569480753, "confusion_matrix": { "negative": { "negative": 45, "neutral": 110, "positive": 27, "__failed__": 0 }, "neutral": { "negative": 99, "neutral": 310, "positive": 181, "__failed__": 0 }, "positive": { "negative": 34, "neutral": 181, "positive": 186, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "kev-08b": { "label": "Kev 0.8B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 177, "accuracy": 0.885, "wilson95": [ 0.8333528310151243, 0.9221362676743804 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 177, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.885, "brier": 0.1989993232506913, "nll": 0.3840450905724345, "ece10": 0.059958916593648985, "calibration_n": 200, "confident_wrong": 11, "macro_f1": 0.8868105946124468, "confusion_matrix": { "world": { "world": 43, "sports": 0, "business": 3, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 47, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 37, "sci_tech": 12, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 0, "sci_tech": 50, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 107, "accuracy": 0.535, "wilson95": [ 0.4658664687236316, 0.6028143584299599 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 107, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.535, "brier": 0.6245781426352852, "nll": 1.3094388143657703, "ece10": 0.06456623656725705, "calibration_n": 200, "confident_wrong": 2, "macro_f1": 0.4124194540471519, "confusion_matrix": { "sadness": { "sadness": 41, "joy": 2, "love": 3, "anger": 6, "fear": 1, "surprise": 4, "__failed__": 0 }, "joy": { "sadness": 18, "joy": 43, "love": 11, "anger": 2, "fear": 4, "surprise": 1, "__failed__": 0 }, "love": { "sadness": 2, "joy": 6, "love": 3, "anger": 1, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 8, "joy": 3, "love": 1, "anger": 12, "fear": 1, "surprise": 3, "__failed__": 0 }, "fear": { "sadness": 6, "joy": 3, "love": 2, "anger": 1, "fear": 7, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 139, "accuracy": 0.695, "wilson95": [ 0.6280144790202163, 0.7546358436926496 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 139, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.695, "brier": 0.41052549696873974, "nll": 0.5996083123730029, "ece10": 0.07492536830774149, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.6737880691997112, "confusion_matrix": { "false": { "false": 95, "true": 14, "__failed__": 0 }, "true": { "false": 47, "true": 44, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 132, "accuracy": 0.66, "wilson95": [ 0.5918849552683626, 0.7220845402909121 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 132, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.66, "brier": 0.4734583353971598, "nll": 0.6873639887842011, "ece10": 0.13153983805715722, "calibration_n": 200, "confident_wrong": 5, "macro_f1": 0.5635990245154665, "confusion_matrix": { "true": { "true": 19, "false": 58, "__failed__": 0 }, "false": { "true": 10, "false": 113, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.4152899534005294, "nll": 0.7306913874851558, "ece10": 0.1696376171397298, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.6809523809523809, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 4, "accuracy": 0.3333333333333333, "wilson95": [ 0.13812009109121315, 0.6093779111272004 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3333333333333333, "brier": 0.7595170133961816, "nll": 1.398127246329658, "ece10": 0.2592347593898994, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.20909090909090908, "confusion_matrix": { "maintenance": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 1, "review": 1, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 1, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 3, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.10426184137892763, "nll": 0.2291473375720531, "ece10": 0.11657350049467752, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9142857142857143, "confusion_matrix": { "design": { "design": 2, "inspection": 0, "purchase": 1, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.22039595041360183, "nll": 0.4446901113524229, "ece10": 0.23896338768999406, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9142857142857143, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 2, "work": 1, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 92, "accuracy": 0.92, "wilson95": [ 0.8500189229905127, 0.9589065385156194 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 92, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.92, "brier": 0.1315786007952973, "nll": 0.262174546347939, "ece10": 0.04762847140448609, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.9191676670668267, "confusion_matrix": { "label_000": { "label_000": 22, "label_001": 3, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 21, "label_002": 1, "label_003": 2, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 24, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 45, "accuracy": 0.45, "wilson95": [ 0.3561453797951198, 0.5475539700255787 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 45, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.6855933703177068, "nll": 1.4014917209018938, "ece10": 0.07922613375573653, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.44685533420827533, "confusion_matrix": { "label_000": { "label_000": 8, "label_001": 5, "label_002": 0, "label_003": 1, "label_004": 2, "label_005": 1, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 13, "label_002": 1, "label_003": 0, "label_004": 3, "label_005": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 5, "label_003": 6, "label_004": 4, "label_005": 2, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 3, "label_003": 7, "label_004": 6, "label_005": 1, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 3, "label_002": 2, "label_003": 2, "label_004": 8, "label_005": 1, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 2, "label_002": 5, "label_003": 1, "label_004": 3, "label_005": 4, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 87, "accuracy": 0.87, "wilson95": [ 0.7901964855992357, 0.9224283257275948 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 87, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.3221044727659176, "nll": 0.8750841858510173, "ece10": 0.2952608316142175, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8592592592592591, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 1, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 2, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 1, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 2, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 2, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 2, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.020879197305772337, "nll": 0.07727602640724165, "ece10": 0.06889411534029663, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 41, "accuracy": 0.36936936936936937, "wilson95": [ 0.2853601624988851, 0.46211779278137066 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 41, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.36936936936936937, "brier": 0.7158561901521011, "nll": 1.2376403051994402, "ece10": 0.1761268584355413, "calibration_n": 111, "confident_wrong": 2 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 58, "accuracy": 0.8055555555555556, "wilson95": [ 0.6996724105411147, 0.8804852062054944 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 58, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8055555555555556, "cluster_bootstrap95": [ 0.6944444444444444, 0.9027777777777778 ], "independent_source_groups": 36, "brier": 0.33079717584900786, "nll": 0.5911762900815459, "ece10": 0.15387356711054076, "calibration_n": 72, "confident_wrong": 1 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 14, "accuracy": 0.7368421052631579, "wilson95": [ 0.5120844910218489, 0.8819358765321748 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7368421052631579, "brier": 0.3128453316511502, "nll": 0.487597936497274, "ece10": 0.11961560756024969, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.7368421052631579, "confusion_matrix": { "false": { "false": 7, "true": 4, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.30433293760132135, "nll": 0.485105684958348, "ece10": 0.20150172952167678, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8117647058823529, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 2, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 16, "accuracy": 0.7272727272727273, "wilson95": [ 0.5184826598759762, 0.8684923971201053 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7272727272727273, "brier": 0.39574722730273754, "nll": 0.5819454965400739, "ece10": 0.13127928862171803, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.717948717948718, "confusion_matrix": { "false": { "false": 6, "true": 4, "__failed__": 0 }, "true": { "false": 2, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.1115883193858396, "nll": 0.27682873431070676, "ece10": 0.22809845619719887, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.41752448925936453, "nll": 0.8009615974969021, "ece10": 0.3888167328442211, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8261904761904761, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.1573199275190077, "nll": 0.35520804812893875, "ece10": 0.21768479290477832, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9377777777777778, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 1, "noise": 0, "permits": 0, "utilities": 2, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.4881219168818126, "nll": 0.9657048615519602, "ece10": 0.38136022489124966, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8551587301587302, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 1, "performance": 0, "security": 3, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 2, "refactor": 0, "nit": 2, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 10, "accuracy": 0.6666666666666666, "wilson95": [ 0.41713547738519774, 0.8482367556028526 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.42854656299949706, "nll": 0.8851359077434435, "ece10": 0.25813155961466355, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.6492063492063492, "confusion_matrix": { "feature": { "feature": 1, "fix": 1, "refactor": 1, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 1, "refactor": 1, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 1, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 2, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 0, "chore": 3, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.37242597988511267, "nll": 0.5546592775769756, "ece10": 0.17895964845226708, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8131868131868132, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.4216643956853427, "nll": 0.6149569582794139, "ece10": 0.20429933428033384, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.746031746031746, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 9, "accuracy": 0.5, "wilson95": [ 0.29031021505169974, 0.7096897849483003 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6134424068085559, "nll": 1.1125381482459866, "ece10": 0.1587848849763735, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.39428571428571424, "confusion_matrix": { "harassment": { "harassment": 1, "threat": 0, "self_harm": 0, "spam": 0, "none": 4, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 1, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 1, "none": 3, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 6, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 15, "accuracy": 0.6818181818181818, "wilson95": [ 0.47318597741042745, 0.8363940681864375 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6818181818181818, "brier": 0.38715130256457614, "nll": 0.7775163659287614, "ece10": 0.20874699219165624, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.672063492063492, "confusion_matrix": { "news_report": { "news_report": 3, "opinion": 0, "press_release": 1, "analysis": 1, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 3, "press_release": 0, "analysis": 2, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 3, "feature": 1, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.034897809271220825, "nll": 0.10401583122324784, "ece10": 0.09141639980721893, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.1891551387980736, "nll": 0.4212006055215248, "ece10": 0.2421744914561643, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 1, "damaged": 3, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.6442532213955117, "nll": 0.8801144643132298, "ece10": 0.29760946320411424, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.32, "confusion_matrix": { "false": { "false": 0, "true": 9, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.136382595725986, "nll": 0.26179291242456165, "ece10": 0.19628731459124305, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8433333333333334, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 1, "purchase_order": 1, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "purchase_order": 0, "receipt": 2, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 12, "accuracy": 0.8, "wilson95": [ 0.5481455128483065, 0.9295245065301844 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.3462713409460031, "nll": 0.7593404951601507, "ece10": 0.294584346866803, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7814285714285714, "confusion_matrix": { "meeting_request": { "meeting_request": 1, "question": 1, "complaint": 0, "thanks": 0, "fyi": 1, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 1, "complaint": 2, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.11426330087226479, "nll": 0.23595224314681826, "ece10": 0.13864028682271975, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.41139698677326647, "nll": 0.5997342841338886, "ece10": 0.06642507247033626, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6625, "confusion_matrix": { "false": { "false": 7, "true": 2, "__failed__": 0 }, "true": { "false": 4, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.5749921787898299, "nll": 0.9957524695791082, "ece10": 0.15879934765573087, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.49269480519480524, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.39082877560454876, "nll": 0.6766420003422174, "ece10": 0.29781175598016574, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7835497835497836, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 2, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 14, "accuracy": 0.7, "wilson95": [ 0.4810271816464765, 0.8545227551323956 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.4011938504529245, "nll": 0.7936782472385535, "ece10": 0.20054736649012433, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7011904761904761, "confusion_matrix": { "cheating": { "cheating": 3, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 2, "boosting": 0, "smurfing": 0, "other": 2, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 1, "other": 2, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 5, "accuracy": 0.23809523809523808, "wilson95": [ 0.10628013022636468, 0.4509117368295075 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.23809523809523808, "brier": 1.0003071530078964, "nll": 1.935117193114249, "ece10": 0.40502182265992775, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.07692307692307693, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 3, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.3145959219084645, "nll": 0.5006514618103349, "ece10": 0.2696352625567404, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9352226720647774, "confusion_matrix": { "false": { "false": 6, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.09546878011433366, "nll": 0.25008138243231165, "ece10": 0.20409969714069073, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 10, "accuracy": 0.7142857142857143, "wilson95": [ 0.4535091567010681, 0.8827862135542308 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7142857142857143, "brier": 0.5779592008656955, "nll": 1.0961996452130351, "ece10": 0.31832741122077246, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.5999999999999999, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 2, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 1, "4": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.5132674710268856, "nll": 0.9654756819549087, "ece10": 0.3030632984279485, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7468975468975468, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 2, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 1, "1": 0, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6088383617989993, "nll": 1.1265769976185076, "ece10": 0.25549517262532057, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.4208333333333333, "confusion_matrix": { "0": { "0": 1, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.444008495659864, "nll": 0.6368195544147407, "ece10": 0.28582120429534924, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8210526315789474, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.1082513883052626, "nll": 0.23655082510098382, "ece10": 0.12567280897158598, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9377777777777778, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "sports": 2, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.09903468687155959, "nll": 0.26931510354360094, "ece10": 0.2191076164222421, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.36271256475635283, "nll": 0.5426574010350383, "ece10": 0.12595007813226314, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7638888888888888, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.4292485145266723, "nll": 0.6038291798004785, "ece10": 0.18597830140073157, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.45893719806763283, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.41620225168667524, "nll": 0.7572969730984005, "ece10": 0.2979424292708881, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8094405594405595, "confusion_matrix": { "0": { "0": 3, "1": 2, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.36674860860199615, "nll": 0.7790265035777261, "ece10": 0.2557155499077698, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7656084656084655, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 1, "french": 0, "mexican": 0, "indian": 2, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 1, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 1, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 2, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 7, "accuracy": 0.4117647058823529, "wilson95": [ 0.2161105679021481, 0.6399455735700021 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4117647058823529, "brier": 0.66397852075028, "nll": 0.8879389136685012, "ece10": 0.28259749859161487, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.2916666666666667, "confusion_matrix": { "false": { "false": 0, "true": 8, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.36990351572238356, "nll": 0.7285331805794244, "ece10": 0.26353241631275176, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6930735930735932, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 1, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 2, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 2, "size_or_fit": 1, "changed_mind": 1, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.5948975206656866, "nll": 1.3099322874656598, "ece10": 0.22750029369998065, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6799999999999999, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 1, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.6114155065670831, "nll": 0.882911820384563, "ece10": 0.22023550451887713, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.6190476190476191, "confusion_matrix": { "false": { "false": 4, "true": 6, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 7, "accuracy": 0.4666666666666667, "wilson95": [ 0.24809535365737387, 0.698830199745016 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.6381944609987776, "nll": 0.888192876145953, "ece10": 0.26081723879497276, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.3181818181818182, "confusion_matrix": { "false": { "false": 0, "true": 8, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.586699152674599, "nll": 0.9852277525087059, "ece10": 0.20188196886377127, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5127272727272728, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 2, "sales": 0, "account": 0, "other": 1, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 1, "account": 0, "other": 3, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 3, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.4134504534913824, "nll": 0.602475637495333, "ece10": 0.1859333592421757, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7571428571428571, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 11, "accuracy": 0.5789473684210527, "wilson95": [ 0.3627590353425504, 0.7685810871754576 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5789473684210527, "brier": 0.5205364699106628, "nll": 0.9385030519807376, "ece10": 0.1297046170407022, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.5844017094017093, "confusion_matrix": { "0": { "0": 4, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 2, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.42656433784090203, "nll": 0.6162309314749276, "ece10": 0.13326778117701943, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.39285714285714285, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.22902042795591893, "nll": 0.37980227906783776, "ece10": 0.13150328198158284, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8328173374613003, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 2, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 8, "accuracy": 0.4, "wilson95": [ 0.21880653237281705, 0.6134184992377469 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.6042051552760783, "nll": 1.0792690303182453, "ece10": 0.14385849215281246, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.3886904761904762, "confusion_matrix": { "0": { "0": 1, "1": 5, "2": 1, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 3, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 3, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.1270724189067768, "nll": 0.2844872554862262, "ece10": 0.21841774253158855, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 1, "reminder": 0, "music": 0, "small_talk": 3, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 7, "accuracy": 0.35, "wilson95": [ 0.18119182410108206, 0.5671457233147638 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35, "brier": 0.6037149991459186, "nll": 0.8089596332885755, "ece10": 0.2783309256856604, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.3350383631713555, "confusion_matrix": { "false": { "false": 5, "true": 4, "__failed__": 0 }, "true": { "false": 9, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 12, "accuracy": 0.5454545454545454, "wilson95": [ 0.3465979532073494, 0.7307970581918668 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5454545454545454, "brier": 0.5826640826053971, "nll": 1.0778121245311516, "ece10": 0.11322581033497948, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.49230769230769234, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 3, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 5, "2": 1, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 517, "accuracy": 0.5744444444444444, "wilson95": [ 0.5418932407222805, 0.6063628485423558 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 684, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6374650512581547, "cluster_bootstrap95": [ 0.5089052873251443, 0.6337125125833389 ], "independent_source_groups": 596, "brier": 0.4425825348526097, "nll": 0.7785048562001406, "ece10": 0.08542927608162211, "calibration_n": 1073, "confident_wrong": 0 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 442, "accuracy": 0.7700348432055749, "wilson95": [ 0.7338820060448208, 0.8025973262493231 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 796, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8504273504273504, "brier": 0.24450247007420747, "nll": 0.49190353478668275, "ece10": 0.07040692168717542, "calibration_n": 936, "confident_wrong": 9 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 378, "accuracy": 0.54, "wilson95": [ 0.5029608116915077, 0.5766025605682475 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 722, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6636029411764706, "cluster_bootstrap95": [ 0.5021638846175223, 0.5761512840607669 ], "independent_source_groups": 650, "brier": 0.45926098466688187, "nll": 0.8617414131981453, "ece10": 0.12289228142234923, "calibration_n": 1088, "confident_wrong": 2 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 19, "accuracy": 0.0475, "wilson95": [ 0.030617076962711882, 0.07299154940982647 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 921, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4605, "brier": 0.606055730465325, "nll": 1.0613050029005968, "ece10": 0.040789748840259564, "calibration_n": 2000, "confident_wrong": 0 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.06289929281234655, "nll": 0.17356363654887336, "ece10": 0.1438080844220545, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 1, "inspection": 0, "purchase": 0, "work": 6, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.10459829467713412, "nll": 0.2407058195880495, "ece10": 0.15809057698204676, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "work": { "work": 6, "purchase": 0, "inspection": 0, "design": 1, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.09212638824635891, "nll": 0.23056055363774638, "ece10": 0.1919198587391799, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 23, "accuracy": 0.8214285714285714, "wilson95": [ 0.6440857521065705, 0.9212149805439572 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8214285714285714, "brier": 0.246034006996186, "nll": 0.4650478936405971, "ece10": 0.08164342477507305, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.8176039646627882, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 6, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 0, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 2, "purchase": 0, "inspection": 1, "design": 4, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 33, "accuracy": 0.825, "wilson95": [ 0.6805000966821229, 0.9125458625396408 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 33, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.825, "brier": 0.3078688215519617, "nll": 0.5737202589631702, "ece10": 0.27582681751250565, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8400828157349898, "confusion_matrix": { "review": { "review": 8, "it": 0, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 2, "it": 6, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 2, "it": 0, "documents": 6, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 2, "it": 0, "documents": 0, "procurement": 6, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 1, "it": 0, "documents": 0, "procurement": 0, "maintenance": 7, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 33, "accuracy": 0.825, "wilson95": [ 0.6805000966821229, 0.9125458625396408 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 33, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.825, "brier": 0.3296419320823526, "nll": 0.6023053805423838, "ece10": 0.24332475659018876, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8355128205128206, "confusion_matrix": { "maintenance": { "maintenance": 7, "procurement": 0, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 5, "documents": 0, "it": 0, "review": 3, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 7, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 7, "review": 1, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 0, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 28, "accuracy": 0.7, "wilson95": [ 0.5456998118185507, 0.8192515477025347 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.4877952288777642, "nll": 0.9417241217929366, "ece10": 0.1407758077605386, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7206446886446887, "confusion_matrix": { "review": { "review": 7, "it": 0, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 3, "it": 5, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 4, "it": 0, "documents": 4, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 2, "it": 0, "documents": 1, "procurement": 5, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 1, "it": 0, "documents": 0, "procurement": 0, "maintenance": 7, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 29, "accuracy": 0.725, "wilson95": [ 0.5716504419378943, 0.8389198375233267 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 29, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.725, "brier": 0.4971845731476364, "nll": 0.9612713993198165, "ece10": 0.1721236848801065, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7402564102564104, "confusion_matrix": { "maintenance": { "maintenance": 7, "procurement": 0, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 5, "documents": 1, "it": 0, "review": 2, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 4, "it": 0, "review": 4, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 6, "review": 2, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 1, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 220, "accuracy": 0.55, "wilson95": [ 0.5010009985806122, 0.5980477719859582 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 220, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.5913233193293541, "nll": 1.2414249275374805, "ece10": 0.058031391114231834, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.4469216387614532, "confusion_matrix": { "sadness": { "sadness": 86, "joy": 7, "love": 9, "anger": 7, "fear": 6, "surprise": 4, "__failed__": 0 }, "joy": { "sadness": 22, "joy": 70, "love": 22, "anger": 0, "fear": 0, "surprise": 4, "__failed__": 0 }, "love": { "sadness": 10, "joy": 9, "love": 11, "anger": 3, "fear": 1, "surprise": 3, "__failed__": 0 }, "anger": { "sadness": 23, "joy": 4, "love": 5, "anger": 26, "fear": 1, "surprise": 5, "__failed__": 0 }, "fear": { "sadness": 15, "joy": 2, "love": 2, "anger": 5, "fear": 26, "surprise": 2, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 3, "love": 1, "anger": 1, "fear": 2, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 243, "accuracy": 0.81, "wilson95": [ 0.7617962081751881, 0.8503651496655933 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 243, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.81, "brier": 0.29440282298705717, "nll": 0.7553396005273703, "ece10": 0.17119478978692246, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 356, "accuracy": 0.89, "wilson95": [ 0.8555490895383555, 0.9170313208808931 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 356, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.89, "brier": 0.2390636928892701, "nll": 0.4095238081843181, "ece10": 0.1809489992121648, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.8356676003734828, "confusion_matrix": { "false": { "false": 293, "true": 16, "__failed__": 0 }, "true": { "false": 28, "true": 63, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 358, "accuracy": 0.895, "wilson95": [ 0.8611090149132419, 0.921376272562664 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 358, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.895, "brier": 0.16911473003486774, "nll": 0.36592345469168913, "ece10": 0.0457460009760812, "calibration_n": 400, "confident_wrong": 21, "macro_f1": 0.8858227483577721, "confusion_matrix": { "world": { "world": 84, "sports": 7, "business": 8, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 2, "sports": 119, "business": 0, "sci_tech": 2, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 59, "sci_tech": 11, "__failed__": 0 }, "sci_tech": { "world": 1, "sports": 0, "business": 5, "sci_tech": 96, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 265, "accuracy": 0.6625, "wilson95": [ 0.6148101420626579, 0.7070983622786957 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 265, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6625, "brier": 0.48080706633725323, "nll": 0.6995985814304262, "ece10": 0.08600208811727303, "calibration_n": 400, "confident_wrong": 11, "macro_f1": 0.5628202948534233, "confusion_matrix": { "true": { "true": 37, "false": 116, "__failed__": 0 }, "false": { "true": 19, "false": 228, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 156, "accuracy": 0.52, "wilson95": [ 0.4635710820137406, 0.5759231991372775 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 156, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.604125648732362, "nll": 1.4011881610319288, "ece10": 0.131688993825557, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 277, "accuracy": 0.6925, "wilson95": [ 0.6456251644125997, 0.7357126022686962 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 277, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6925, "brier": 0.405163180049237, "nll": 0.5928301835623019, "ece10": 0.06840595645449328, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.685661165737067, "confusion_matrix": { "false": { "false": 168, "true": 20, "__failed__": 0 }, "true": { "false": 103, "true": 109, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 281, "accuracy": 0.7025, "wilson95": [ 0.6559450648860605, 0.7452024559085495 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 281, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7025, "brier": 0.4035533687904679, "nll": 0.5919643778454001, "ece10": 0.08112618965264849, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.683087104441441, "confusion_matrix": { "false": { "false": 190, "true": 10, "__failed__": 0 }, "true": { "false": 109, "true": 91, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 67, "accuracy": 0.67, "wilson95": [ 0.5730534379509052, 0.7543687726587197 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 67, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.67, "brier": 0.4255299903637497, "nll": 0.6166762774598002, "ece10": 0.061802433825587665, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6643271284711627, "confusion_matrix": { "yes": { "yes": 27, "no": 27, "__failed__": 0 }, "no": { "yes": 6, "no": 40, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 45, "accuracy": 0.45, "wilson95": [ 0.3561453797951198, 0.5475539700255787 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 45, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.7245398664094137, "nll": 1.7735325464168614, "ece10": 0.12396388489744878, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.4736152465883503, "confusion_matrix": { "balance": { "balance": 3, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 2, "dispute_charge": 0, "loan_payment": 2, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 3, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 5, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 }, "statement": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 4, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 3, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 1, "fraud_alert": 0, "branch_appointment": 4, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 3, "mortgage": 0, "wire": 1, "statement": 2, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 4, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 4, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 4, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 3, "limit_increase": 0, "other": 1, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 3, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 1, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 4, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 140, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.43245299927334097, "nll": 0.709480706803212, "ece10": 0.12459998626989506, "calibration_n": 200, "confident_wrong": 1 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 37, "accuracy": 0.37, "wilson95": [ 0.28182360534324524, 0.4677947041905709 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 117, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.585, "brier": 0.5862579050387045, "nll": 1.1888234054367177, "ece10": 0.12068951108556897, "calibration_n": 200, "confident_wrong": 4 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 74, "accuracy": 0.74, "wilson95": [ 0.6462901055081282, 0.8159530153525186 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 74, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.74, "brier": 0.38017253966179637, "nll": 0.9458518000823892, "ece10": 0.08586964806430084, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.7186354867700367, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 2, "receipt": 12, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 2, "receipt": 3, "contract": 0, "resume": 0, "bank_statement": 9, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 2, "bank_statement": 0, "clinical_note": 5, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 2, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 9, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 2, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 6, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 7, "accuracy": 0.07, "wilson95": [ 0.03431926106727268, 0.137495147390735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 206, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.515, "brier": 0.6581908051991305, "nll": 1.2350106632897906, "ece10": 0.15642551719367154, "calibration_n": 400, "confident_wrong": 22 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 53, "accuracy": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 53, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.53, "brier": 0.5910906071795524, "nll": 1.3640804688043173, "ece10": 0.14763295966728487, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5227752639517346, "confusion_matrix": { "politics": { "politics": 0, "business": 2, "technology": 3, "science": 2, "health": 4, "sports": 0, "entertainment": 1, "world": 2, "culture": 0, "weather": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 8, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 1, "technology": 8, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 1, "technology": 1, "science": 6, "health": 1, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 1, "health": 8, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "science": 2, "health": 0, "sports": 5, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 0, "business": 0, "technology": 1, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 2, "weather": 1, "__failed__": 0 }, "world": { "politics": 0, "business": 5, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 3, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 1, "science": 0, "health": 1, "sports": 0, "entertainment": 1, "world": 2, "culture": 5, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 2, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 5, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 32, "accuracy": 0.32, "wilson95": [ 0.23669147324999093, 0.41662618610452384 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 32, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.32, "brier": 0.8084104413875451, "nll": 1.9056264265539458, "ece10": 0.06402405175165424, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.2740776614689658, "confusion_matrix": { "biology": { "biology": 1, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 3, "mathematics": 1, "earth_science": 0, "engineering": 1, "economics": 0, "other": 1, "__failed__": 0 }, "medicine": { "biology": 0, "medicine": 8, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 2, "mathematics": 0, "earth_science": 1, "engineering": 3, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 5, "computer_science": 4, "mathematics": 0, "earth_science": 4, "engineering": 2, "economics": 1, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 5, "mathematics": 0, "earth_science": 2, "engineering": 7, "economics": 1, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 2, "mathematics": 1, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 0, "medicine": 4, "chemistry": 0, "physics": 0, "computer_science": 2, "mathematics": 0, "earth_science": 5, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 1, "earth_science": 1, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 1, "engineering": 1, "economics": 3, "other": 0, "__failed__": 0 }, "other": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 4, "mathematics": 1, "earth_science": 1, "engineering": 4, "economics": 1, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 7, "accuracy": 0.07, "wilson95": [ 0.03431926106727268, 0.137495147390735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 64, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.32, "brier": 0.6826609455771994, "nll": 1.3619600476377878, "ece10": 0.23390427496116042, "calibration_n": 100, "confident_wrong": 8 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 12, "accuracy": 0.12, "wilson95": [ 0.06999406437019488, 0.19812099426711421 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 95, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.475, "brier": 0.2747243526612381, "nll": 0.47771928595079033, "ece10": 0.06613650816383736, "calibration_n": 100, "confident_wrong": 2 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 86, "accuracy": 0.86, "wilson95": [ 0.7786281178903616, 0.9147365634006086 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.2396362253999651, "nll": 0.47442591005838053, "ece10": 0.04937874486208649, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8606480584092524, "confusion_matrix": { "negative": { "negative": 34, "neutral": 2, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 6, "neutral": 27, "positive": 0, "__failed__": 0 }, "positive": { "negative": 1, "neutral": 5, "positive": 25, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 15, "accuracy": 0.15, "wilson95": [ 0.0930598528365663, 0.23283559590832364 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 92, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.46, "brier": 0.3058348307342398, "nll": 0.4329014000727976, "ece10": 0.14229683738947466, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 39, "accuracy": 0.39, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 212, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7066666666666667, "brier": 0.38065564102481875, "nll": 0.7212911382329024, "ece10": 0.05752621983907159, "calibration_n": 300, "confident_wrong": 2 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 52, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.6785851248820048, "nll": 1.822788369186754, "ece10": 0.1650608470783607, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5143725968648329, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 2, "password_reset": 3, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 3, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "product_question": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 1, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 2, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "speak_to_human": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 1, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 2, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 7, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 2, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 5, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 2, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 4, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 1, "other": 2, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 37, "accuracy": 0.37, "wilson95": [ 0.28182360534324524, 0.4677947041905709 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.37, "brier": 0.7757980199689066, "nll": 1.808729804120494, "ece10": 0.13198354421182126, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.3774721513627602, "confusion_matrix": { "billing": { "billing": 4, "account_access": 0, "shipping": 1, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 2, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 5, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 5, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 2, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 2, "account_access": 1, "shipping": 2, "returns": 0, "technical_outage": 2, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 2, "__failed__": 0 }, "onboarding": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 2, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 1, "__failed__": 0 }, "integrations": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 2, "onboarding": 0, "integrations": 3, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 1, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 1, "__failed__": 0 }, "compliance": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 2, "pricing": 0, "data_privacy": 1, "other": 1, "__failed__": 0 }, "pricing": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 2, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 1, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "other": { "billing": 2, "account_access": 0, "shipping": 3, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 1, "pricing": 0, "data_privacy": 1, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 7, "accuracy": 0.07, "wilson95": [ 0.03431926106727268, 0.137495147390735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 129, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43, "brier": 0.708210288922861, "nll": 1.4716554673632103, "ece10": 0.11671812991677395, "calibration_n": 300, "confident_wrong": 1 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 53, "accuracy": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 53, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.53, "brier": 0.620006941125407, "nll": 1.3843897955230113, "ece10": 0.13738970671623865, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5281355409296585, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 1, "__failed__": 0 }, "change_flight": { "new_booking": 1, "change_flight": 4, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 2, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 4, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 3, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 4, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 5, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 7, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 3, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "visa_docs": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 2, "loyalty": 2, "disruption": 0, "other": 1, "__failed__": 0 }, "loyalty": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 3, "disruption": 1, "other": 2, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 4, "other": 2, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 1, "visa_docs": 0, "loyalty": 2, "disruption": 2, "other": 6, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 95, "accuracy": 0.95, "wilson95": [ 0.8882495307680808, 0.9784563208456319 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 95, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.1028314295649141, "nll": 0.23225167612483777, "ece10": 0.04030321223899663, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.9497288277012933, "confusion_matrix": { "business": { "business": 22, "science and technology": 3, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 0, "science and technology": 23, "sports": 1, "world": 1, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 81, "accuracy": 0.81, "wilson95": [ 0.7222115462093562, 0.8748524849023126 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 81, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.81, "brier": 0.2823550489303634, "nll": 0.7812610791666402, "ece10": 0.1664183922662798, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.7833333333333332, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 2, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 2, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 2, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 2, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 1, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 43, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43, "brier": 0.6823105217037261, "nll": 1.3901995536116403, "ece10": 0.09393621845839249, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.4389962353441714, "confusion_matrix": { "anger": { "anger": 5, "fear": 2, "joy": 0, "love": 0, "sadness": 7, "surprise": 3, "__failed__": 0 }, "fear": { "anger": 0, "fear": 9, "joy": 1, "love": 0, "sadness": 6, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 6, "love": 5, "sadness": 4, "surprise": 2, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 3, "love": 7, "sadness": 7, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 2, "joy": 3, "love": 1, "sadness": 10, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 1, "joy": 4, "love": 0, "sadness": 4, "surprise": 6, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2114, "accuracy": 0.7170963364993216, "wilson95": [ 0.7005629931830261, 0.7330646314223209 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2114, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7170963364993216, "brier": 0.41322898034414446, "nll": 0.9540681912584, "ece10": 0.09154437308248844, "calibration_n": 2948, "confident_wrong": 3, "macro_f1": 0.7356683145806086, "confusion_matrix": { "alarm": { "alarm": 62, "audio": 0, "calendar": 11, "cooking": 0, "datetime": 8, "email": 0, "general": 2, "iot": 13, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 44, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 12, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 1, "audio": 0, "calendar": 318, "cooking": 0, "datetime": 38, "email": 3, "general": 20, "iot": 0, "lists": 1, "music": 0, "news": 3, "play": 0, "qa": 0, "recommendation": 10, "social": 2, "takeaway": 4, "transport": 0, "weather": 0, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 65, "datetime": 0, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 96, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 4, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 3, "email": 204, "general": 39, "iot": 0, "lists": 10, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 2, "social": 2, "takeaway": 0, "transport": 1, "weather": 2, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 2, "datetime": 8, "email": 1, "general": 123, "iot": 3, "lists": 1, "music": 1, "news": 1, "play": 6, "qa": 10, "recommendation": 8, "social": 7, "takeaway": 2, "transport": 5, "weather": 3, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 4, "calendar": 0, "cooking": 9, "datetime": 5, "email": 0, "general": 23, "iot": 159, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 11, "transport": 1, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 8, "cooking": 0, "datetime": 0, "email": 0, "general": 19, "iot": 0, "lists": 112, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 10, "iot": 0, "lists": 1, "music": 62, "news": 0, "play": 5, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 2, "general": 24, "iot": 0, "lists": 0, "music": 0, "news": 92, "play": 0, "qa": 1, "recommendation": 0, "social": 3, "takeaway": 1, "transport": 1, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 2, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 50, "iot": 5, "lists": 2, "music": 55, "news": 0, "play": 265, "qa": 3, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 16, "email": 1, "general": 110, "iot": 1, "lists": 0, "music": 1, "news": 2, "play": 7, "qa": 113, "recommendation": 16, "social": 5, "takeaway": 0, "transport": 10, "weather": 4, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 9, "email": 0, "general": 10, "iot": 0, "lists": 0, "music": 0, "news": 5, "play": 9, "qa": 0, "recommendation": 55, "social": 0, "takeaway": 0, "transport": 1, "weather": 2, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 4, "general": 15, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 84, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 2, "email": 0, "general": 7, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 46, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 14, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 98, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 3, "email": 0, "general": 28, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 116, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1719, "accuracy": 0.5831071913161465, "wilson95": [ 0.5652123118708426, 0.6007857631733918 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1719, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5831071913161465, "brier": 0.5536144703518091, "nll": 1.3537345154830704, "ece10": 0.05818642548453965, "calibration_n": 2948, "confident_wrong": 1, "macro_f1": 0.624907211415549, "confusion_matrix": { "alarm": { "alarm": 76, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 8, "email": 0, "general": 3, "iot": 8, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 29, "calendar": 0, "cooking": 0, "datetime": 4, "email": 0, "general": 18, "iot": 3, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 4, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 1, "audio": 0, "calendar": 219, "cooking": 1, "datetime": 72, "email": 8, "general": 57, "iot": 5, "lists": 6, "music": 0, "news": 8, "play": 0, "qa": 3, "recommendation": 10, "social": 3, "takeaway": 3, "transport": 3, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 60, "datetime": 0, "email": 0, "general": 7, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 91, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 4, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 3, "email": 221, "general": 37, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 1, "social": 1, "takeaway": 0, "transport": 2, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 3, "cooking": 6, "datetime": 10, "email": 1, "general": 123, "iot": 5, "lists": 2, "music": 3, "news": 3, "play": 5, "qa": 6, "recommendation": 6, "social": 3, "takeaway": 0, "transport": 6, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 8, "datetime": 3, "email": 0, "general": 49, "iot": 142, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 1, "recommendation": 0, "social": 1, "takeaway": 7, "transport": 1, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 2, "datetime": 0, "email": 0, "general": 14, "iot": 3, "lists": 111, "music": 4, "news": 1, "play": 1, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 3, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 12, "iot": 0, "lists": 0, "music": 60, "news": 0, "play": 6, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 1, "email": 2, "general": 41, "iot": 0, "lists": 1, "music": 0, "news": 68, "play": 1, "qa": 2, "recommendation": 2, "social": 3, "takeaway": 1, "transport": 1, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 15, "calendar": 1, "cooking": 1, "datetime": 4, "email": 0, "general": 108, "iot": 12, "lists": 15, "music": 95, "news": 2, "play": 126, "qa": 2, "recommendation": 1, "social": 3, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 2, "calendar": 0, "cooking": 0, "datetime": 16, "email": 1, "general": 131, "iot": 6, "lists": 0, "music": 1, "news": 3, "play": 8, "qa": 84, "recommendation": 13, "social": 3, "takeaway": 0, "transport": 14, "weather": 5, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 8, "cooking": 0, "datetime": 5, "email": 0, "general": 14, "iot": 1, "lists": 0, "music": 1, "news": 2, "play": 14, "qa": 0, "recommendation": 42, "social": 0, "takeaway": 1, "transport": 6, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 6, "general": 25, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 71, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 3, "email": 0, "general": 5, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 44, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 11, "email": 0, "general": 15, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 93, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 22, "email": 1, "general": 48, "iot": 4, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 8, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 7, "weather": 59, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 757, "accuracy": 0.6453537936913896, "wilson95": [ 0.6175423110079907, 0.6722163454738381 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 757, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6453537936913896, "cluster_bootstrap95": [ 0.6132943757544123, 0.6760930166194529 ], "independent_source_groups": 47, "brier": 0.505683544396779, "nll": 0.8845215182966842, "ece10": 0.05147356147742794, "calibration_n": 1173, "confident_wrong": 1, "macro_f1": 0.579831792209992, "confusion_matrix": { "negative": { "negative": 68, "neutral": 107, "positive": 7, "__failed__": 0 }, "neutral": { "negative": 56, "neutral": 497, "positive": 37, "__failed__": 0 }, "positive": { "negative": 19, "neutral": 190, "positive": 192, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "kev-4b": { "label": "Kev 4B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 179, "accuracy": 0.895, "wilson95": [ 0.8448196800887696, 0.9302925120731899 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 179, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.895, "brier": 0.1767876030761775, "nll": 0.34244315490025584, "ece10": 0.03687743823586764, "calibration_n": 200, "confident_wrong": 6, "macro_f1": 0.8975002515501637, "confusion_matrix": { "world": { "world": 45, "sports": 0, "business": 1, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 40, "sci_tech": 9, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 2, "sci_tech": 48, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 114, "accuracy": 0.57, "wilson95": [ 0.5007048776415268, 0.6366567766656557 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 114, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.57, "brier": 0.606967400984179, "nll": 1.3394369665202592, "ece10": 0.07971726439001425, "calibration_n": 200, "confident_wrong": 6, "macro_f1": 0.46730913107724703, "confusion_matrix": { "sadness": { "sadness": 40, "joy": 5, "love": 1, "anger": 6, "fear": 4, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 17, "joy": 43, "love": 10, "anger": 3, "fear": 3, "surprise": 3, "__failed__": 0 }, "love": { "sadness": 3, "joy": 3, "love": 5, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 }, "anger": { "sadness": 5, "joy": 3, "love": 0, "anger": 16, "fear": 1, "surprise": 3, "__failed__": 0 }, "fear": { "sadness": 4, "joy": 2, "love": 2, "anger": 1, "fear": 9, "surprise": 2, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 186, "accuracy": 0.93, "wilson95": [ 0.8859451911438936, 0.9578478281716571 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 186, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.93, "brier": 0.1812120540808574, "nll": 0.32122924371155703, "ece10": 0.15999994407352783, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.9281609195402298, "confusion_matrix": { "false": { "false": 109, "true": 0, "__failed__": 0 }, "true": { "false": 14, "true": 77, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 182, "accuracy": 0.91, "wilson95": [ 0.8622342812328346, 0.9423125511378069 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 182, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.91, "brier": 0.16193803084804423, "nll": 0.2889934093067367, "ece10": 0.08488241919848426, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.9004314636574842, "confusion_matrix": { "true": { "true": 60, "false": 17, "__failed__": 0 }, "false": { "true": 1, "false": 122, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.30390060530982765, "nll": 0.6040621184469385, "ece10": 0.24679392750424678, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.7733333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 1, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.46546714781067805, "nll": 0.8913756166695984, "ece10": 0.24558570831714874, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.7, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 2, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.020806253146741994, "nll": 0.10811775783496226, "ece10": 0.10112404854285717, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.034735572743726253, "nll": 0.15231520242284088, "ece10": 0.13875435635615938, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 95, "accuracy": 0.95, "wilson95": [ 0.8882495307680808, 0.9784563208456319 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 95, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.07434302173297898, "nll": 0.1542642159917574, "ece10": 0.04965625415241845, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.9497288277012933, "confusion_matrix": { "label_000": { "label_000": 22, "label_001": 3, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 23, "label_002": 1, "label_003": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 45, "accuracy": 0.45, "wilson95": [ 0.3561453797951198, 0.5475539700255787 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 45, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.7130170685271352, "nll": 1.5867646251948204, "ece10": 0.173407318498611, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.4621142833601121, "confusion_matrix": { "label_000": { "label_000": 7, "label_001": 3, "label_002": 1, "label_003": 0, "label_004": 4, "label_005": 2, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 9, "label_002": 0, "label_003": 0, "label_004": 5, "label_005": 2, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 7, "label_003": 3, "label_004": 5, "label_005": 2, "__failed__": 0 }, "label_003": { "label_000": 1, "label_001": 0, "label_002": 2, "label_003": 5, "label_004": 9, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 4, "label_004": 10, "label_005": 2, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 0, "label_002": 3, "label_003": 0, "label_004": 5, "label_005": 7, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.22561721256907077, "nll": 0.5553706538315091, "ece10": 0.1838768561967553, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8657407407407407, "confusion_matrix": { "label_000": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 1, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 2, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 2, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 2, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 1, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 2, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0033835316028775065, "nll": 0.021980490067187313, "ece10": 0.020821523426282372, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 60, "accuracy": 0.5405405405405406, "wilson95": [ 0.4480286067541221, 0.6303403036761436 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 60, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5405405405405406, "brier": 0.5674221951201202, "nll": 0.9622386579317557, "ece10": 0.11211486413658894, "calibration_n": 111, "confident_wrong": 1 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 67, "accuracy": 0.9305555555555556, "wilson95": [ 0.8475202051632649, 0.9699746184342299 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 67, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9305555555555556, "cluster_bootstrap95": [ 0.8333333333333334, 1.0 ], "independent_source_groups": 36, "brier": 0.15975443240129236, "nll": 0.3138037374345566, "ece10": 0.14769202436453605, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 17, "accuracy": 0.8947368421052632, "wilson95": [ 0.6860591728889099, 0.9706414397011299 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8947368421052632, "brier": 0.22188843658525928, "nll": 0.3828638857616994, "ece10": 0.1796708846204394, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8944444444444444, "confusion_matrix": { "false": { "false": 9, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.09683460925217208, "nll": 0.22278616729197545, "ece10": 0.1915023868166614, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 21, "accuracy": 0.9545454545454546, "wilson95": [ 0.7820195416134267, 0.991930572378736 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9545454545454546, "brier": 0.161514348304574, "nll": 0.3045504969410923, "ece10": 0.15206283577812707, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9536842105263157, "confusion_matrix": { "false": { "false": 9, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.029790701152839064, "nll": 0.09858157646826682, "ece10": 0.08818800127112039, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.3202864385565721, "nll": 0.5734249018898258, "ece10": 0.25048997262006534, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8261904761904761, "confusion_matrix": { "0": { "0": 2, "1": 2, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02833457101784491, "nll": 0.12018122656363417, "ece10": 0.10915422954743415, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.29690874792717353, "nll": 0.5780960753372237, "ece10": 0.2307099265609212, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8571428571428571, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 2, "refactor": 0, "nit": 1, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 2, "refactor": 0, "nit": 2, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.2624045814637508, "nll": 0.572349244364854, "ece10": 0.3839207852261339, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9111111111111111, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 2, "test": 0, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.118899264227391, "nll": 0.2495734236693781, "ece10": 0.20750556298721667, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8785714285714286, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.22125783553768688, "nll": 0.3845692436571091, "ece10": 0.1201015637320567, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.20355718176963475, "nll": 0.44202463733900216, "ece10": 0.20818779839880305, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8825396825396824, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 1, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 6, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 18, "accuracy": 0.8181818181818182, "wilson95": [ 0.6148339256287345, 0.9269311541657794 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8181818181818182, "brier": 0.2353179608820298, "nll": 0.4749063633674662, "ece10": 0.19031257661747367, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.7894017094017094, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 4, "press_release": 0, "analysis": 1, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 3, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 1, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.016983416578438465, "nll": 0.06064451237808169, "ece10": 0.054894150303603256, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.008658266829503521, "nll": 0.06670148268647137, "ece10": 0.06357730878672337, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.09580232652757527, "nll": 0.19972210520757086, "ece10": 0.1602163557751476, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9411764705882353, "confusion_matrix": { "false": { "false": 8, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00834145471596314, "nll": 0.0513065024262152, "ece10": 0.04809942174517284, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.1560249898834048, "nll": 0.3240150726572097, "ece10": 0.1362107687607695, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.12499081500252883, "nll": 0.2228727954774247, "ece10": 0.11029107768330514, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.21830356734323264, "nll": 0.3538623169570432, "ece10": 0.22530854632156433, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6989966555183946, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.4244253751340459, "nll": 0.6652992689076367, "ece10": 0.2399139882080949, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6363636363636362, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 5, "3": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.15245669205136453, "nll": 0.32154398608418155, "ece10": 0.2493260909317173, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9474747474747475, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 16, "accuracy": 0.8, "wilson95": [ 0.5839825677481064, 0.919342337420202 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.2855304890861715, "nll": 0.5523129678171625, "ece10": 0.20266510863108061, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8171428571428571, "confusion_matrix": { "cheating": { "cheating": 3, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 2, "other": 1, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 11, "accuracy": 0.5238095238095238, "wilson95": [ 0.3236953458284443, 0.7165599389846583 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5238095238095238, "brier": 0.6515719701618142, "nll": 1.2239791036795693, "ece10": 0.16946824685634598, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.5342857142857144, "confusion_matrix": { "spelling": { "spelling": 1, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 3, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 0, "punctuation": 0, "none": 1, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 0, "none": 3, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 2, "none": 3, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.05021651235584597, "nll": 0.15045700993204789, "ece10": 0.1353387326083506, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02549494808660024, "nll": 0.11828507574346174, "ece10": 0.10771285134756156, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 12, "accuracy": 0.8571428571428571, "wilson95": [ 0.6005862050302965, 0.9599060787285351 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.31348141286111586, "nll": 0.6265179623819572, "ece10": 0.235270723659277, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.75, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.26387813311393316, "nll": 0.5193074487310183, "ece10": 0.2959893364992463, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9444444444444444, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.3708023330784609, "nll": 0.6870007251395056, "ece10": 0.24172583836849193, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6568181818181819, "confusion_matrix": { "0": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.31815989699789843, "nll": 0.5021990393550629, "ece10": 0.1674550341175543, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8235294117647058, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.09826167838603794, "nll": 0.2106287887317378, "ece10": 0.1363682170221659, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "politics": { "politics": 3, "business": 0, "technology": 0, "sports": 0, "culture": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.26225189695390977, "nll": 0.5063954150783969, "ece10": 0.20061215095365814, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7754545454545454, "confusion_matrix": { "payments": { "payments": 1, "web": 1, "infra": 2, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 3, "infra": 1, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.27520046078958615, "nll": 0.4496834923516331, "ece10": 0.2082421595379346, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8819444444444444, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.05316035481388645, "nll": 0.13782268169770684, "ece10": 0.12108074149050174, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.23021693781846903, "nll": 0.4223202469469992, "ece10": 0.23624368815065325, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9393939393939394, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.06966905430325118, "nll": 0.18686957233777635, "ece10": 0.09390060501878525, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571427, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.33266490247442443, "nll": 0.5022517448891128, "ece10": 0.2180114671578705, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7638888888888888, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.16631328910958723, "nll": 0.31697674534950915, "ece10": 0.16804565995276474, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.838095238095238, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 0, "changed_mind": 3, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.3947496992323722, "nll": 1.17927205905932, "ece10": 0.20756812808239009, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7785714285714286, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.26007437956039714, "nll": 0.4085100920107924, "ece10": 0.1146949366450615, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8117647058823529, "confusion_matrix": { "false": { "false": 7, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 11, "accuracy": 0.7333333333333333, "wilson95": [ 0.4804956594401943, 0.8910254667430761 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7333333333333333, "brier": 0.3062065449292132, "nll": 0.4782422605750282, "ece10": 0.15849905359039418, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7222222222222222, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 3, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.282291515608117, "nll": 0.5549096252958212, "ece10": 0.23474792359455549, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8732467532467533, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 1, "tech": 0, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.1903174357152062, "nll": 0.35470791928458995, "ece10": 0.2364081653488708, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9411764705882353, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 17, "accuracy": 0.8947368421052632, "wilson95": [ 0.6860591728889099, 0.9706414397011299 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8947368421052632, "brier": 0.31566935377759014, "nll": 0.5815501183097238, "ece10": 0.2660907601156468, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8244949494949495, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.1901155664339378, "nll": 0.3569882139333605, "ece10": 0.24585406073616653, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9328063241106719, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.0692248851055075, "nll": 0.16784608703488105, "ece10": 0.14398693666825396, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 18, "accuracy": 0.9, "wilson95": [ 0.6989663547715128, 0.9721335187862319 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.31348093714308484, "nll": 0.6053412209334164, "ece10": 0.29338923422802304, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8950549450549451, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.07469405687335978, "nll": 0.21510962283554771, "ece10": 0.15282375769585885, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 12, "accuracy": 0.6, "wilson95": [ 0.3865815007622531, 0.781193467627183 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.38773596069279453, "nll": 0.5563825630400799, "ece10": 0.19539365825878205, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.5604395604395604, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 8, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 18, "accuracy": 0.8181818181818182, "wilson95": [ 0.6148339256287345, 0.9269311541657794 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8181818181818182, "brier": 0.30236326901824634, "nll": 0.6069431003622018, "ece10": 0.23553573150390328, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8113247863247863, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 3, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 640, "accuracy": 0.7111111111111111, "wilson95": [ 0.6806516924621175, 0.7397760233629707 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 810, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7548928238583411, "cluster_bootstrap95": [ 0.6762578729046651, 0.7400251993185536 ], "independent_source_groups": 596, "brier": 0.3417202240406545, "nll": 0.6017155327162111, "ece10": 0.07632483481268172, "calibration_n": 1073, "confident_wrong": 2 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 488, "accuracy": 0.8501742160278746, "wilson95": [ 0.8186534438801792, 0.8770391096109363 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 848, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.905982905982906, "brier": 0.17080522768049697, "nll": 0.3568868596952932, "ece10": 0.10387016122919619, "calibration_n": 936, "confident_wrong": 5 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 507, "accuracy": 0.7242857142857143, "wilson95": [ 0.6900251297683105, 0.7560980646881738 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 876, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8051470588235294, "cluster_bootstrap95": [ 0.6914259296890876, 0.7574489453280014 ], "independent_source_groups": 650, "brier": 0.27744134594451464, "nll": 0.510128955994002, "ece10": 0.08558888773023476, "calibration_n": 1088, "confident_wrong": 5 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 60, "accuracy": 0.15, "wilson95": [ 0.11834499530337544, 0.18831361073063213 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1340, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.67, "brier": 0.46462189737943815, "nll": 0.8140179268813924, "ece10": 0.09581151549258066, "calibration_n": 2000, "confident_wrong": 2 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.049507120897283895, "nll": 0.16121449569404497, "ece10": 0.14088893370000205, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "wilson95": [ 0.7735463743160095, 0.9801879358846942 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 26, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.0719556440005571, "nll": 0.19013226221820773, "ece10": 0.15359612633799136, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9283882783882784, "confusion_matrix": { "work": { "work": 6, "purchase": 0, "inspection": 0, "design": 1, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 0, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0723114440489014, "nll": 0.2122742369877019, "ece10": 0.17939546746568685, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.09808025455090894, "nll": 0.2536232409560498, "ece10": 0.17153591672456986, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 1, "purchase": 6, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.25937808273497015, "nll": 0.5315014352395853, "ece10": 0.33254440746406316, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9498039215686275, "confusion_matrix": { "review": { "review": 7, "it": 0, "documents": 0, "procurement": 1, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 7, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.27586260358386305, "nll": 0.5604441022780084, "ece10": 0.27824249902944453, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8757422969187676, "confusion_matrix": { "maintenance": { "maintenance": 6, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 7, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 1, "review": 6, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 37, "accuracy": 0.925, "wilson95": [ 0.8013576647568946, 0.9741639742254119 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.3088429753567429, "nll": 0.6239181070860386, "ece10": 0.3522553675651558, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.923235294117647, "confusion_matrix": { "review": { "review": 6, "it": 0, "documents": 1, "procurement": 1, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 1, "it": 0, "documents": 7, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.343129695936392, "nll": 0.6659006946731642, "ece10": 0.3019304543620103, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8715686274509805, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 7, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 7, "review": 1, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 1, "it": 1, "review": 5, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 224, "accuracy": 0.56, "wilson95": [ 0.5110127871951399, 0.6078457374847447 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 224, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.56, "brier": 0.5700663062606436, "nll": 1.2351968425552793, "ece10": 0.08415560959930404, "calibration_n": 400, "confident_wrong": 2, "macro_f1": 0.48686129527915784, "confusion_matrix": { "sadness": { "sadness": 82, "joy": 8, "love": 9, "anger": 7, "fear": 10, "surprise": 3, "__failed__": 0 }, "joy": { "sadness": 20, "joy": 62, "love": 20, "anger": 3, "fear": 4, "surprise": 9, "__failed__": 0 }, "love": { "sadness": 11, "joy": 6, "love": 14, "anger": 2, "fear": 2, "surprise": 2, "__failed__": 0 }, "anger": { "sadness": 21, "joy": 1, "love": 4, "anger": 33, "fear": 2, "surprise": 3, "__failed__": 0 }, "fear": { "sadness": 13, "joy": 2, "love": 1, "anger": 2, "fear": 30, "surprise": 4, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 3, "love": 0, "anger": 1, "fear": 1, "surprise": 3, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 254, "accuracy": 0.8466666666666667, "wilson95": [ 0.8015339429418983, 0.8830335970090831 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 254, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8466666666666667, "brier": 0.2164649545021624, "nll": 0.5624979892410693, "ece10": 0.13957567550981098, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 294, "accuracy": 0.735, "wilson95": [ 0.6896629392432349, 0.7758662824196457 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 294, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.735, "brier": 0.326141741606435, "nll": 0.4829816612069433, "ece10": 0.08242316811343098, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.7037037037037037, "confusion_matrix": { "false": { "false": 212, "true": 97, "__failed__": 0 }, "true": { "false": 9, "true": 82, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 370, "accuracy": 0.925, "wilson95": [ 0.8949522937386957, 0.9469622560771523 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 370, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.1467093627476386, "nll": 0.3118116165903906, "ece10": 0.03510000906739542, "calibration_n": 400, "confident_wrong": 11, "macro_f1": 0.9183547846111487, "confusion_matrix": { "world": { "world": 87, "sports": 6, "business": 8, "sci_tech": 2, "__failed__": 0 }, "sports": { "world": 2, "sports": 121, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 1, "sports": 0, "business": 65, "sci_tech": 6, "__failed__": 0 }, "sci_tech": { "world": 0, "sports": 0, "business": 5, "sci_tech": 97, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 361, "accuracy": 0.9025, "wilson95": [ 0.8694877208462789, 0.9278548822146123 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 361, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9025, "brier": 0.16501618268501364, "nll": 0.29404303078986865, "ece10": 0.09007141919956524, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.8930327278711456, "confusion_matrix": { "true": { "true": 121, "false": 32, "__failed__": 0 }, "false": { "true": 7, "false": 240, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 214, "accuracy": 0.7133333333333334, "wilson95": [ 0.6597183711272249, 0.7615539611503023 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 214, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7133333333333334, "brier": 0.45370434226852724, "nll": 1.049046452692203, "ece10": 0.18332549438509593, "calibration_n": 300, "confident_wrong": 0 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 365, "accuracy": 0.9125, "wilson95": [ 0.8807392673857086, 0.9364130897884968 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 365, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9125, "brier": 0.17968762769856605, "nll": 0.3221826742785438, "ece10": 0.14990052458505257, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.9124556806883485, "confusion_matrix": { "false": { "false": 187, "true": 1, "__failed__": 0 }, "true": { "false": 34, "true": 178, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 299, "accuracy": 0.7475, "wilson95": [ 0.7027083556965948, 0.7875830586079285 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 299, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7475, "brier": 0.3212773914908436, "nll": 0.4854912078312269, "ece10": 0.041287916953743266, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.7372511527780488, "confusion_matrix": { "false": { "false": 189, "true": 11, "__failed__": 0 }, "true": { "false": 90, "true": 110, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 74, "accuracy": 0.74, "wilson95": [ 0.6462901055081282, 0.8159530153525186 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 74, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.74, "brier": 0.29836976725957953, "nll": 0.4639454631216842, "ece10": 0.10368577542047736, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.7291666666666667, "confusion_matrix": { "yes": { "yes": 47, "no": 7, "__failed__": 0 }, "no": { "yes": 19, "no": 27, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 43, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43, "brier": 0.7109203061785155, "nll": 1.6434733970563427, "ece10": 0.15001469422266617, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.49054773521685285, "confusion_matrix": { "balance": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 3, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 2, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 4, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 3, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 2, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 5, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 3, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 4, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 4, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 4, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 5, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 3, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 4, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 5, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 4, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 2, "limit_increase": 0, "other": 2, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 3, "other": 1, "__failed__": 0 }, "other": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 5, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 46, "accuracy": 0.46, "wilson95": [ 0.36560808309024106, 0.5573513967663177 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 139, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.695, "brier": 0.39603550955525313, "nll": 0.6828292766854931, "ece10": 0.06336127811613844, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 38, "accuracy": 0.38, "wilson95": [ 0.29097599252478734, 0.47790244704488916 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 120, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5106762073957082, "nll": 1.0398228593367063, "ece10": 0.11221931590264496, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 74, "accuracy": 0.74, "wilson95": [ 0.6462901055081282, 0.8159530153525186 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 74, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.74, "brier": 0.3475777523692278, "nll": 0.8197168572747893, "ece10": 0.10757441546356139, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.689601467827274, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 13, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 12, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 6, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 2, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 1, "receipt": 1, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 6, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 5, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 4, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 1, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 4, "accuracy": 0.04, "wilson95": [ 0.01566330399154762, 0.0983707143588792 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 200, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6461361057084537, "nll": 1.220192282862069, "ece10": 0.16877652490420397, "calibration_n": 400, "confident_wrong": 8 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 65, "accuracy": 0.65, "wilson95": [ 0.5525444329688796, 0.7363575175690247 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 65, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.526003363533787, "nll": 1.1855461330349137, "ece10": 0.11058333674140178, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6325868575868576, "confusion_matrix": { "politics": { "politics": 0, "business": 0, "technology": 3, "science": 3, "health": 4, "sports": 0, "entertainment": 0, "world": 2, "culture": 1, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 1, "technology": 9, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 1, "technology": 0, "science": 7, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 6, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 0, "technology": 1, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 2, "weather": 0, "__failed__": 0 }, "world": { "politics": 0, "business": 2, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 5, "culture": 1, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 8, "weather": 1, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 6, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 37, "accuracy": 0.37, "wilson95": [ 0.28182360534324524, 0.4677947041905709 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.37, "brier": 0.7543703299708953, "nll": 1.7357075781749705, "ece10": 0.1013889760647236, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.3267944200380923, "confusion_matrix": { "biology": { "biology": 4, "medicine": 1, "chemistry": 1, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 2, "economics": 0, "other": 1, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 9, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 2, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 2, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 1, "medicine": 3, "chemistry": 0, "physics": 4, "computer_science": 1, "mathematics": 0, "earth_science": 4, "engineering": 2, "economics": 1, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 2, "chemistry": 1, "physics": 0, "computer_science": 5, "mathematics": 0, "earth_science": 2, "engineering": 5, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 2, "mathematics": 0, "earth_science": 2, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 1, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 7, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 1, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 2, "engineering": 0, "economics": 2, "other": 1, "__failed__": 0 }, "other": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 2, "computer_science": 2, "mathematics": 1, "earth_science": 3, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 73, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.365, "brier": 0.5518751083179854, "nll": 1.1106967640807954, "ece10": 0.15642856942076386, "calibration_n": 100, "confident_wrong": 8 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 30, "accuracy": 0.3, "wilson95": [ 0.2189488529493276, 0.3958485463334666 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 119, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.595, "brier": 0.2785510853322638, "nll": 0.4898674566687349, "ece10": 0.06294949658861493, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 85, "accuracy": 0.85, "wilson95": [ 0.7671644040916763, 0.9069401471634337 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 85, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.210223888387487, "nll": 0.4120813095940601, "ece10": 0.0583592534157319, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.849890350877193, "confusion_matrix": { "negative": { "negative": 33, "neutral": 3, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 6, "neutral": 25, "positive": 2, "__failed__": 0 }, "positive": { "negative": 1, "neutral": 3, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 15, "accuracy": 0.15, "wilson95": [ 0.0930598528365663, 0.23283559590832364 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 96, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.24195640455227363, "nll": 0.3620959751881796, "ece10": 0.0644272069733786, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 51, "accuracy": 0.51, "wilson95": [ 0.4134801193402717, 0.6057800106955886 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 236, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7866666666666666, "brier": 0.3010305459018191, "nll": 0.5779552149484585, "ece10": 0.034378724921625844, "calibration_n": 300, "confident_wrong": 4 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 53, "accuracy": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 53, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.53, "brier": 0.6475617915695834, "nll": 1.654982342034922, "ece10": 0.14958222905474788, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.564985131983828, "confusion_matrix": { "order_status": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 2, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 2, "password_reset": 2, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 2, "__failed__": 0 }, "shipping_delay": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 3, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 2, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 2, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 4, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "speak_to_human": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 3, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 3, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 2, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 4, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 2, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 2, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 2, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 1, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 2, "__failed__": 0 }, "partner_inquiry": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 2, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 1, "other": 2, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 2, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 36, "accuracy": 0.36, "wilson95": [ 0.2727122040140433, 0.4576459754839126 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 36, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.36, "brier": 0.7754057677249974, "nll": 1.8043316370904288, "ece10": 0.09562880408625989, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.3619258940950117, "confusion_matrix": { "billing": { "billing": 3, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 4, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 4, "shipping": 1, "returns": 1, "technical_outage": 1, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 3, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 1, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 4, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 2, "__failed__": 0 }, "technical_outage": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 4, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 5, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 2, "integrations": 4, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 7, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 1, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 2, "pricing": 1, "data_privacy": 1, "other": 2, "__failed__": 0 }, "pricing": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 4, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 2, "__failed__": 0 }, "other": { "billing": 0, "account_access": 0, "shipping": 3, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 4, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 12, "accuracy": 0.12, "wilson95": [ 0.06999406437019488, 0.19812099426711421 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 152, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5066666666666667, "brier": 0.6219589778911563, "nll": 1.3671281045362353, "ece10": 0.08742909773013414, "calibration_n": 300, "confident_wrong": 0 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 59, "accuracy": 0.59, "wilson95": [ 0.4920143005341078, 0.6813268697886348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 59, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.59, "brier": 0.6347139540015602, "nll": 1.481566876164082, "ece10": 0.08429079203392717, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.590310511928159, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 3, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 4, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 4, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 11, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 5, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 2, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 8, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 3, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 2, "loyalty": 1, "disruption": 0, "other": 4, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 3, "disruption": 2, "other": 2, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 6, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 1, "car": 1, "visa_docs": 1, "loyalty": 1, "disruption": 2, "other": 5, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 95, "accuracy": 0.95, "wilson95": [ 0.8882495307680808, 0.9784563208456319 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 95, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.08518476575722786, "nll": 0.1855729269380998, "ece10": 0.059710481168231355, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.9497288277012933, "confusion_matrix": { "business": { "business": 22, "science and technology": 3, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 0, "science and technology": 23, "sports": 1, "world": 1, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.21838452369743297, "nll": 0.5438586979537132, "ece10": 0.16247136310656846, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8532407407407407, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 2, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 2, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 2, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 2, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 1, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 2, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 2, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 1, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.7092492736334289, "nll": 1.5403223704742275, "ece10": 0.1619963792310614, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.4920859419321763, "confusion_matrix": { "anger": { "anger": 8, "fear": 2, "joy": 1, "love": 1, "sadness": 3, "surprise": 2, "__failed__": 0 }, "fear": { "anger": 1, "fear": 10, "joy": 1, "love": 0, "sadness": 4, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 8, "love": 2, "sadness": 5, "surprise": 2, "__failed__": 0 }, "love": { "anger": 1, "fear": 0, "joy": 2, "love": 5, "sadness": 9, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 0, "joy": 2, "love": 1, "sadness": 10, "surprise": 3, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 0, "joy": 3, "love": 0, "sadness": 5, "surprise": 7, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2327, "accuracy": 0.7893487109905021, "wilson95": [ 0.7742571570337314, 0.8036871613855828 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2327, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7893487109905021, "brier": 0.3488869979226008, "nll": 0.8181420358914114, "ece10": 0.19128307243858092, "calibration_n": 2948, "confident_wrong": 0, "macro_f1": 0.7963428817728981, "confusion_matrix": { "alarm": { "alarm": 79, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 5, "email": 0, "general": 1, "iot": 5, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 45, "calendar": 5, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 1, "audio": 0, "calendar": 363, "cooking": 1, "datetime": 2, "email": 0, "general": 4, "iot": 0, "lists": 1, "music": 0, "news": 3, "play": 0, "qa": 16, "recommendation": 7, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 66, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 1, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 83, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 9, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 0, "datetime": 1, "email": 220, "general": 13, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 25, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 14, "cooking": 8, "datetime": 0, "email": 0, "general": 74, "iot": 1, "lists": 0, "music": 1, "news": 0, "play": 3, "qa": 64, "recommendation": 10, "social": 4, "takeaway": 0, "transport": 4, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 0, "calendar": 2, "cooking": 2, "datetime": 1, "email": 0, "general": 8, "iot": 193, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 2, "social": 0, "takeaway": 3, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 1, "lists": 117, "music": 4, "news": 0, "play": 0, "qa": 5, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 69, "news": 0, "play": 1, "qa": 5, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 101, "play": 0, "qa": 20, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 4, "calendar": 1, "cooking": 0, "datetime": 2, "email": 0, "general": 25, "iot": 4, "lists": 0, "music": 118, "news": 1, "play": 214, "qa": 12, "recommendation": 2, "social": 1, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 1, "news": 1, "play": 1, "qa": 270, "recommendation": 3, "social": 1, "takeaway": 0, "transport": 2, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 1, "news": 6, "play": 1, "qa": 20, "recommendation": 61, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 2, "general": 15, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 82, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 0, "email": 0, "general": 4, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 8, "recommendation": 2, "social": 0, "takeaway": 40, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 7, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 114, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 3, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 10, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 136, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2123, "accuracy": 0.7201492537313433, "wilson95": [ 0.7036654046690269, 0.7360601084073328 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2123, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7201492537313433, "brier": 0.43171828464229284, "nll": 1.0338484857923793, "ece10": 0.18733028442632327, "calibration_n": 2948, "confident_wrong": 0, "macro_f1": 0.7282483020028239, "confusion_matrix": { "alarm": { "alarm": 69, "audio": 1, "calendar": 7, "cooking": 0, "datetime": 2, "email": 0, "general": 2, "iot": 14, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 2, "audio": 40, "calendar": 5, "cooking": 0, "datetime": 1, "email": 0, "general": 7, "iot": 1, "lists": 0, "music": 2, "news": 0, "play": 1, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 3, "audio": 0, "calendar": 346, "cooking": 0, "datetime": 5, "email": 1, "general": 9, "iot": 1, "lists": 2, "music": 0, "news": 1, "play": 0, "qa": 20, "recommendation": 9, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 66, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 83, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 10, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 1, "datetime": 1, "email": 207, "general": 24, "iot": 1, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 23, "recommendation": 1, "social": 1, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 17, "cooking": 7, "datetime": 1, "email": 1, "general": 72, "iot": 3, "lists": 0, "music": 2, "news": 1, "play": 2, "qa": 63, "recommendation": 6, "social": 2, "takeaway": 0, "transport": 6, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 2, "audio": 1, "calendar": 1, "cooking": 11, "datetime": 3, "email": 0, "general": 5, "iot": 177, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 7, "social": 0, "takeaway": 4, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 1, "calendar": 5, "cooking": 1, "datetime": 0, "email": 0, "general": 10, "iot": 2, "lists": 112, "music": 5, "news": 0, "play": 0, "qa": 4, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 66, "news": 0, "play": 0, "qa": 7, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 84, "play": 0, "qa": 30, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "play": { "alarm": 1, "audio": 8, "calendar": 3, "cooking": 3, "datetime": 4, "email": 0, "general": 47, "iot": 9, "lists": 3, "music": 145, "news": 3, "play": 137, "qa": 17, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 3, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 4, "email": 0, "general": 9, "iot": 1, "lists": 0, "music": 2, "news": 1, "play": 1, "qa": 259, "recommendation": 2, "social": 1, "takeaway": 0, "transport": 3, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 1, "news": 1, "play": 1, "qa": 23, "recommendation": 61, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 0, "general": 21, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 6, "recommendation": 0, "social": 73, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 2, "datetime": 0, "email": 0, "general": 10, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 12, "recommendation": 2, "social": 0, "takeaway": 30, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 5, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 111, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 9, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 1, "weather": 130, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 867, "accuracy": 0.7391304347826086, "wilson95": [ 0.7132499926289553, 0.7634497325508278 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 867, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7391304347826086, "cluster_bootstrap95": [ 0.7059357926201599, 0.7723896103896104 ], "independent_source_groups": 47, "brier": 0.3737906270646476, "nll": 0.6712534109909841, "ece10": 0.03180461750527944, "calibration_n": 1173, "confident_wrong": 15, "macro_f1": 0.6888414960082129, "confusion_matrix": { "negative": { "negative": 83, "neutral": 91, "positive": 8, "__failed__": 0 }, "neutral": { "negative": 33, "neutral": 521, "positive": 36, "__failed__": 0 }, "positive": { "negative": 9, "neutral": 129, "positive": 263, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "laya-multilingual": { "label": "Laya multilingual", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 183, "accuracy": 0.915, "wilson95": [ 0.8681041292843443, 0.9462542498225244 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 183, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.915, "brier": 0.14643856839273947, "nll": 0.3089276154736638, "ece10": 0.06723148500554985, "calibration_n": 200, "confident_wrong": 10, "macro_f1": 0.9166170281900246, "confusion_matrix": { "world": { "world": 45, "sports": 0, "business": 3, "sci_tech": 2, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 42, "sci_tech": 7, "__failed__": 0 }, "sci_tech": { "world": 0, "sports": 0, "business": 2, "sci_tech": 50, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 103, "accuracy": 0.515, "wilson95": [ 0.44610849192745716, 0.5833261482812249 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 103, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.515, "brier": 0.809284698147068, "nll": 2.764849683649367, "ece10": 0.3556588067362353, "calibration_n": 200, "confident_wrong": 50, "macro_f1": 0.3871846789200841, "confusion_matrix": { "sadness": { "sadness": 40, "joy": 3, "love": 3, "anger": 6, "fear": 4, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 13, "joy": 40, "love": 18, "anger": 2, "fear": 3, "surprise": 3, "__failed__": 0 }, "love": { "sadness": 4, "joy": 2, "love": 6, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 12, "joy": 1, "love": 3, "anger": 10, "fear": 1, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 6, "joy": 1, "love": 3, "anger": 0, "fear": 7, "surprise": 3, "__failed__": 0 }, "surprise": { "sadness": 1, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 200, "accuracy": 1.0, "wilson95": [ 0.9811546736227335, 1.0 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 200, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 5, "head_accuracy": 1.0, "brier": 7.4000000000002e-09, "nll": 6.001850780378344e-06, "ece10": 5.999999999999894e-06, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 109, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 91, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 200, "accuracy": 1.0, "wilson95": [ 0.9811546736227335, 1.0 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 200, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 4, "head_accuracy": 1.0, "brier": 0.0, "nll": 0.0, "ece10": 0.0, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "true": { "true": 77, "false": 0, "__failed__": 0 }, "false": { "true": 0, "false": 123, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": 0.5, "development": { "n": 100, "correct": 100, "wilson95": [ 0.9630065017930143, 1.0 ] }, "test_coverage": 1.0, "accepted_n": 200, "accepted_correct": 200, "accepted_accuracy": 1.0, "accepted_wilson95": [ 0.9811546736227335, 1.0 ], "errors_per_1000_all_test_cases": 0.0, "note": "Selected on development only, applied unchanged to test. An empirical pilot estimate, not a production guarantee." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 6, "accuracy": 0.5, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.8548360013662569, "nll": 4.998867950450753, "ece10": 0.37269584375712506, "calibration_n": 12, "confident_wrong": 3, "macro_f1": 0.5, "confusion_matrix": { "maintenance": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 1, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 1, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 1, "it": 1, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 6, "accuracy": 0.5, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.9087503670810496, "nll": 2.324370230395649, "ece10": 0.5368916783305001, "calibration_n": 12, "confident_wrong": 1, "macro_f1": 0.3466666666666667, "confusion_matrix": { "maintenance": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 3, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.5499266638478593, "nll": 4.0698946858220575, "ece10": 0.28872245292030785, "calibration_n": 12, "confident_wrong": 2, "macro_f1": 0.6595238095238095, "confusion_matrix": { "design": { "design": 1, "inspection": 0, "purchase": 2, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 2, "work": 1, "__failed__": 0 }, "work": { "design": 1, "inspection": 0, "purchase": 0, "work": 2, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.3607632949751615, "nll": 0.6238833030595045, "ece10": 0.27468362636683624, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.830952380952381, "confusion_matrix": { "design": { "design": 2, "inspection": 0, "purchase": 1, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 2, "work": 1, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 91, "accuracy": 0.91, "wilson95": [ 0.837737871472837, 0.9519274599974348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 91, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.91, "brier": 0.22375089179005617, "nll": 0.4634866419576293, "ece10": 0.21343428229470285, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.9088293058630981, "confusion_matrix": { "label_000": { "label_000": 23, "label_001": 2, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 20, "label_002": 2, "label_003": 2, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 24, "label_003": 1, "__failed__": 0 }, "label_003": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 24, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 22, "accuracy": 0.22, "wilson95": [ 0.150012823846054, 0.31070353514985793 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.22, "brier": 0.8611369641134939, "nll": 1.8788848234916093, "ece10": 0.21934912187746514, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.16117355007648235, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 13, "label_002": 1, "label_003": 1, "label_004": 0, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 13, "label_002": 3, "label_003": 0, "label_004": 0, "label_005": 0, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 9, "label_002": 4, "label_003": 2, "label_004": 1, "label_005": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 9, "label_002": 5, "label_003": 3, "label_004": 0, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 3, "label_001": 13, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 2, "label_001": 12, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 2, "accuracy": 0.02, "wilson95": [ 0.00550196755016235, 0.07001179072854388 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 2, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 100, "head_accuracy": 0.02, "brier": 1.0169471690482539, "nll": 4.391013727969089, "ece10": 0.13938411458499564, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.0005446623093681917, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 43, "accuracy": 0.8958333333333334, "wilson95": [ 0.7783257830561368, 0.9546782809581869 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 43, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8958333333333334, "brier": 0.1460864116774011, "nll": 0.24127540313839893, "ece10": 0.04416196943495302, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 37, "accuracy": 0.3333333333333333, "wilson95": [ 0.2525113750360455, 0.4253053265284187 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 44, "head_accuracy": 0.3333333333333333, "brier": 1.0213696226307196, "nll": 2.7072900835390903, "ece10": 0.39403242130798394, "calibration_n": 111, "confident_wrong": 29 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 29, "accuracy": 0.4027777777777778, "wilson95": [ 0.2972111115210491, 0.5181932831504844 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 29, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4027777777777778, "cluster_bootstrap95": [ 0.2777777777777778, 0.5277777777777778 ], "independent_source_groups": 36, "brier": 0.7703613561820865, "nll": 1.3887859814324046, "ece10": 0.3783321526093076, "calibration_n": 72, "confident_wrong": 8 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 11, "accuracy": 0.5789473684210527, "wilson95": [ 0.3627590353425504, 0.7685810871754576 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5789473684210527, "brier": 0.8222848052631578, "nll": 1.9215395303816474, "ece10": 0.4044052631578947, "calibration_n": 19, "confident_wrong": 8, "macro_f1": 0.36666666666666664, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 8, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.5167124987499999, "nll": 0.9684168973618016, "ece10": 0.25701875, "calibration_n": 16, "confident_wrong": 2, "macro_f1": 0.6, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 5, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 12, "accuracy": 0.5454545454545454, "wilson95": [ 0.3465979532073494, 0.7307970581918668 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5454545454545454, "brier": 0.6784860572727273, "nll": 0.9905864364910688, "ece10": 0.33839545454545455, "calibration_n": 22, "confident_wrong": 3, "macro_f1": 0.5089285714285714, "confusion_matrix": { "false": { "false": 9, "true": 1, "__failed__": 0 }, "true": { "false": 9, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.16803685713982994, "nll": 0.6110827783884035, "ece10": 0.10454391622187668, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.9415584415584415, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 3, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 7, "accuracy": 0.3888888888888889, "wilson95": [ 0.2030524657216362, 0.613809583977621 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3888888888888889, "brier": 0.7752036513828189, "nll": 1.6148130001785255, "ece10": 0.23080246673146912, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.33714285714285713, "confusion_matrix": { "0": { "0": 1, "1": 2, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 2, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 3, "3": 0, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 2, "3": 0, "4": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 17, "accuracy": 0.8947368421052632, "wilson95": [ 0.6860591728889099, 0.9706414397011299 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8947368421052632, "brier": 0.20034264718448558, "nll": 0.3946834690010757, "ece10": 0.15463865038470231, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8914285714285715, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 1, "roads": 0, "noise": 0, "permits": 3, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 1, "roads": 0, "noise": 0, "permits": 0, "utilities": 2, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 4, "accuracy": 0.23529411764705882, "wilson95": [ 0.09555044322080117, 0.4726179811956498 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.23529411764705882, "brier": 0.9702224289062283, "nll": 2.0167578786065445, "ece10": 0.37269750411350444, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.2341269841269841, "confusion_matrix": { "bug": { "bug": 1, "performance": 0, "security": 1, "style": 0, "refactor": 0, "nit": 1, "__failed__": 0 }, "performance": { "bug": 1, "performance": 1, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "style": { "bug": 2, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 1, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "nit": { "bug": 1, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 2, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 7, "accuracy": 0.4666666666666667, "wilson95": [ 0.24809535365737387, 0.698830199745016 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.8296051919422606, "nll": 1.7635122176446065, "ece10": 0.43126009431031287, "calibration_n": 15, "confident_wrong": 2, "macro_f1": 0.39285714285714285, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 1, "fix": 1, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 2, "docs": 1, "test": 1, "chore": 0, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.39738016470588233, "nll": 0.5755753871148505, "ece10": 0.28064705882352936, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7571428571428571, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 6, "accuracy": 0.375, "wilson95": [ 0.18481232558863633, 0.6135895945449727 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.375, "brier": 0.7841932025000001, "nll": 1.1361667089372565, "ece10": 0.4053875, "calibration_n": 16, "confident_wrong": 2, "macro_f1": 0.2727272727272727, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 8, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 10, "accuracy": 0.5555555555555556, "wilson95": [ 0.33716415642042774, 0.7544048187299437 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5555555555555556, "brier": 0.563943355679467, "nll": 1.4542052327334438, "ece10": 0.12585620465239536, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.29545454545454547, "confusion_matrix": { "harassment": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 2, "none": 3, "__failed__": 0 }, "threat": { "harassment": 1, "threat": 0, "self_harm": 0, "spam": 1, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 6, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 12, "accuracy": 0.5454545454545454, "wilson95": [ 0.3465979532073494, 0.7307970581918668 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5454545454545454, "brier": 0.5935777979525636, "nll": 1.1927416014795156, "ece10": 0.2115944659682174, "calibration_n": 22, "confident_wrong": 1, "macro_f1": 0.4761904761904761, "confusion_matrix": { "news_report": { "news_report": 1, "opinion": 2, "press_release": 0, "analysis": 2, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 4, "press_release": 0, "analysis": 1, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 1, "press_release": 3, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 1, "opinion": 0, "press_release": 0, "analysis": 3, "feature": 0, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.273942635675351, "nll": 0.7924930099580894, "ece10": 0.14318765288293536, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.7214285714285714, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 1, "confidentiality": 0, "termination": 2, "indemnification": 0, "force_majeure": 0, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 15, "accuracy": 0.7894736842105263, "wilson95": [ 0.5666572155618583, 0.9149232336708375 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7894736842105263, "brier": 0.3191648903482911, "nll": 0.6491136835486592, "ece10": 0.10593751793248046, "calibration_n": 19, "confident_wrong": 1, "macro_f1": 0.7833333333333333, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 3, "weather_delay": 0, "failed_attempt": 0, "damaged": 2, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 1, "damaged": 2, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.6039697447058823, "nll": 0.8213835093972983, "ece10": 0.2143, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.3952569169960475, "confusion_matrix": { "false": { "false": 1, "true": 8, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.35278260916847903, "nll": 2.847159154793149, "ece10": 0.2664304844187715, "calibration_n": 15, "confident_wrong": 2, "macro_f1": 0.8333333333333333, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 2, "receipt": 1, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 9, "accuracy": 0.6, "wilson95": [ 0.357468301206246, 0.8017550385865841 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5773445347135113, "nll": 3.1973740787471896, "ece10": 0.25648137653014713, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.5533333333333333, "confusion_matrix": { "meeting_request": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 1, "fyi": 2, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 2, "complaint": 0, "thanks": 1, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 2, "thanks": 0, "fyi": 1, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 1, "fyi": 2, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.35078035153588116, "nll": 0.6592178401662083, "ece10": 0.15544497768888982, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.6198412698412699, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 1, "travel": 4, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 3, "software": 1, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 2, "travel": 0, "office_supplies": 1, "software": 0, "client_entertainment": 0, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 10, "accuracy": 0.5555555555555556, "wilson95": [ 0.33716415642042774, 0.7544048187299437 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5555555555555556, "brier": 0.7987434088888887, "nll": 1.6100109014821267, "ece10": 0.43337777777777775, "calibration_n": 18, "confident_wrong": 6, "macro_f1": 0.4461538461538461, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 8, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 4, "accuracy": 0.25, "wilson95": [ 0.10182067491213048, 0.49498316535508774 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.25, "brier": 0.8755593062937362, "nll": 1.8057446712491534, "ece10": 0.2581513163085132, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.20896464646464646, "confusion_matrix": { "0": { "0": 0, "1": 2, "2": 1, "3": 1, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 2, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 2, "2": 3, "3": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 8, "accuracy": 0.4444444444444444, "wilson95": [ 0.24559518127005633, 0.6628358435795723 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4444444444444444, "brier": 0.6475502242616792, "nll": 1.022602107217262, "ece10": 0.2582422937248119, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.2896825396825397, "confusion_matrix": { "0": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 5, "2": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 9, "accuracy": 0.45, "wilson95": [ 0.2581978582556243, 0.6579146575496577 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.6208735923188531, "nll": 1.184554342639878, "ece10": 0.2079536452822865, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.4398989898989899, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 2, "boosting": 0, "smurfing": 0, "other": 2, "__failed__": 0 }, "boosting": { "cheating": 1, "harassment": 0, "boosting": 1, "smurfing": 1, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 1, "harassment": 0, "boosting": 1, "smurfing": 1, "other": 0, "__failed__": 0 }, "other": { "cheating": 1, "harassment": 0, "boosting": 0, "smurfing": 3, "other": 1, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 5, "accuracy": 0.23809523809523808, "wilson95": [ 0.10628013022636468, 0.4509117368295075 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.23809523809523808, "brier": 1.2997689892470345, "nll": 3.290342784014924, "ece10": 0.6553718963351, "calibration_n": 21, "confident_wrong": 6, "macro_f1": 0.07692307692307693, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 3, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 7, "accuracy": 0.4375, "wilson95": [ 0.23098652405492354, 0.6682144360118811 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4375, "brier": 0.9477130275, "nll": 1.8240493160271927, "ece10": 0.4971, "calibration_n": 16, "confident_wrong": 5, "macro_f1": 0.30434782608695654, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 9, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 18, "accuracy": 0.782608695652174, "wilson95": [ 0.5809651698373599, 0.9033602197341379 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.782608695652174, "brier": 0.32474193145456665, "nll": 0.7102760402021684, "ece10": 0.14761852881148962, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 0.7765656565656566, "confusion_matrix": { "plumbing": { "plumbing": 4, "electrical": 1, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 4, "hvac": 0, "handyman": 1, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 1, "electrical": 0, "hvac": 2, "handyman": 0, "appliance_repair": 1, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 1, "hvac": 0, "handyman": 4, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 7, "accuracy": 0.5, "wilson95": [ 0.26799202452413634, 0.7320079754758637 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.7039272526415322, "nll": 1.4544553968075085, "ece10": 0.23819782901212116, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.2583333333333333, "confusion_matrix": { "0": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 2, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 1, "2": 0, "3": 1, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 5, "accuracy": 0.3125, "wilson95": [ 0.14164643854782039, 0.5559564416525933 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3125, "brier": 0.9617099813697908, "nll": 2.414808340715633, "ece10": 0.4262750187481252, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.23026315789473684, "confusion_matrix": { "0": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.7989498935848762, "nll": 1.6543819769668042, "ece10": 0.41117867861328683, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.44999999999999996, "confusion_matrix": { "0": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 2, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 2, "2": 0, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.5663715870588235, "nll": 0.8032481302109628, "ece10": 0.2258294117647059, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.6136363636363636, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.27698612942856893, "nll": 0.6260535883125542, "ece10": 0.13972352397264698, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.8006349206349206, "confusion_matrix": { "politics": { "politics": 1, "business": 1, "technology": 1, "sports": 0, "culture": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.4283155545270971, "nll": 0.7614199633814926, "ece10": 0.2253016123628397, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.47142857142857136, "confusion_matrix": { "payments": { "payments": 3, "web": 0, "infra": 1, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 2, "infra": 2, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 1, "infra": 3, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 2, "data": 1, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 2, "data": 0, "model": 0, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.3967619882352941, "nll": 0.6945632407059691, "ece10": 0.23991764705882349, "calibration_n": 17, "confident_wrong": 3, "macro_f1": 0.7571428571428571, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.48296089500000006, "nll": 0.7330985434304039, "ece10": 0.221575, "calibration_n": 16, "confident_wrong": 2, "macro_f1": 0.6536796536796536, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6172059386875937, "nll": 0.9877010515194978, "ece10": 0.08616554469553045, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5151515151515151, "confusion_matrix": { "0": { "0": 2, "1": 3, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 2, "__failed__": 0 }, "2": { "0": 0, "1": 3, "2": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 5, "accuracy": 0.2777777777777778, "wilson95": [ 0.12499753379553993, 0.5087265656029746 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2777777777777778, "brier": 0.7965705131795132, "nll": 1.790152315056775, "ece10": 0.28063073734458516, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.218013468013468, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 3, "french": 0, "mexican": 0, "indian": 0, "thai": 1, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 1, "indian": 0, "thai": 1, "middle_eastern": 1, "__failed__": 0 }, "indian": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 3, "middle_eastern": 0, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 4, "accuracy": 0.23529411764705882, "wilson95": [ 0.09555044322080117, 0.4726179811956498 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.23529411764705882, "brier": 1.0638470423529416, "nll": 1.664753986533838, "ece10": 0.5619764705882353, "calibration_n": 17, "confident_wrong": 4, "macro_f1": 0.19047619047619047, "confusion_matrix": { "false": { "false": 0, "true": 8, "__failed__": 0 }, "true": { "false": 5, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 5, "accuracy": 0.2777777777777778, "wilson95": [ 0.12499753379553993, 0.5087265656029746 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2777777777777778, "brier": 1.0111056148469522, "nll": 2.393447288997143, "ece10": 0.526084633966124, "calibration_n": 18, "confident_wrong": 4, "macro_f1": 0.17411764705882354, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 3, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 3, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 2, "wrong_item": 1, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 1, "late_delivery": 3, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 4, "accuracy": 0.2222222222222222, "wilson95": [ 0.09000928108601694, 0.45214584316212625 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2222222222222222, "brier": 1.2996595463330434, "nll": 5.354078495659616, "ece10": 0.6723719976550091, "calibration_n": 18, "confident_wrong": 8, "macro_f1": 0.1388888888888889, "confusion_matrix": { "0": { "0": 2, "1": 2, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 1, "1": 2, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 4, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "4": { "0": 2, "1": 2, "2": 0, "3": 0, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.958185725, "nll": 3.979378948379363, "ece10": 0.5056625, "calibration_n": 16, "confident_wrong": 7, "macro_f1": 0.4920634920634921, "confusion_matrix": { "false": { "false": 3, "true": 7, "__failed__": 0 }, "true": { "false": 1, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 4, "accuracy": 0.26666666666666666, "wilson95": [ 0.10897453325692374, 0.5195043405598057 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.26666666666666666, "brier": 0.9128888146666666, "nll": 1.254464384973881, "ece10": 0.4883933333333334, "calibration_n": 15, "confident_wrong": 2, "macro_f1": 0.25339366515837103, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 4, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.5247441461404324, "nll": 2.886144162417418, "ece10": 0.23065780918904868, "calibration_n": 17, "confident_wrong": 3, "macro_f1": 0.6033333333333333, "confusion_matrix": { "billing": { "billing": 4, "tech": 0, "sales": 1, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 1, "tech": 0, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 1, "sales": 0, "account": 2, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 1, "sales": 1, "account": 0, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.6414459070588234, "nll": 0.9507358767439233, "ece10": 0.38268823529411766, "calibration_n": 17, "confident_wrong": 3, "macro_f1": 0.43333333333333335, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 8, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 8, "accuracy": 0.42105263157894735, "wilson95": [ 0.23141891282454238, 0.6372409646574496 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42105263157894735, "brier": 0.6675177489586113, "nll": 1.2302405517244865, "ece10": 0.16265452501433472, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.3333333333333333, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 4, "1": 0, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 1, "1": 3, "2": 1, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.6451866376470589, "nll": 0.9538350794209374, "ece10": 0.2794235294117647, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.43333333333333335, "confusion_matrix": { "false": { "false": 8, "true": 3, "__failed__": 0 }, "true": { "false": 5, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 8, "accuracy": 0.4444444444444444, "wilson95": [ 0.24559518127005633, 0.6628358435795723 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4444444444444444, "brier": 0.8736426300000001, "nll": 1.5652775009970012, "ece10": 0.4254277777777778, "calibration_n": 18, "confident_wrong": 6, "macro_f1": 0.375, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 9, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 9, "accuracy": 0.45, "wilson95": [ 0.2581978582556243, 0.6579146575496577 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.8118372452723424, "nll": 1.9169007302371817, "ece10": 0.10025775662467756, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.3173076923076923, "confusion_matrix": { "0": { "0": 0, "1": 6, "2": 0, "3": 1, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 2, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 14, "accuracy": 0.7, "wilson95": [ 0.4810271816464765, 0.8545227551323956 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.3819043794793833, "nll": 0.715040353264999, "ece10": 0.19918989181524893, "calibration_n": 20, "confident_wrong": 2, "macro_f1": 0.7, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 1, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 1, "information_request": 1, "reminder": 2, "music": 0, "small_talk": 1, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 1, "music": 0, "small_talk": 3, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 7, "accuracy": 0.35, "wilson95": [ 0.18119182410108206, 0.5671457233147638 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35, "brier": 0.7419315939999999, "nll": 1.0206707919568754, "ece10": 0.33131, "calibration_n": 20, "confident_wrong": 1, "macro_f1": 0.3350383631713555, "confusion_matrix": { "false": { "false": 5, "true": 4, "__failed__": 0 }, "true": { "false": 9, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 7, "accuracy": 0.3181818181818182, "wilson95": [ 0.1636059318135624, 0.5268140225895723 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3181818181818182, "brier": 0.8805863357969529, "nll": 1.6520770610919, "ece10": 0.3643315733268157, "calibration_n": 22, "confident_wrong": 1, "macro_f1": 0.20869565217391306, "confusion_matrix": { "0": { "0": 0, "1": 4, "2": 1, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 3, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 5, "2": 1, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 411, "accuracy": 0.45666666666666667, "wilson95": [ 0.4243763917188884, 0.4893252876643829 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 522, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 47, "head_accuracy": 0.4864864864864865, "cluster_bootstrap95": [ 0.4109156735371691, 0.4954216778062591 ], "independent_source_groups": 596, "brier": 0.8026675210891009, "nll": 2.2360956146421582, "ece10": 0.3259154644159501, "calibration_n": 1073, "confident_wrong": 236 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 175, "accuracy": 0.3048780487804878, "wilson95": [ 0.26861772372603143, 0.34373269422916786 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 436, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 159, "head_accuracy": 0.4658119658119658, "brier": 0.7902806395332392, "nll": 2.8849935747119866, "ece10": 0.29536924862974734, "calibration_n": 936, "confident_wrong": 150 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 116, "accuracy": 0.1657142857142857, "wilson95": [ 0.1400089139479363, 0.19506861788125296 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 320, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 100, "head_accuracy": 0.29411764705882354, "cluster_bootstrap95": [ 0.13912638594742743, 0.19433217666575348 ], "independent_source_groups": 650, "brier": 0.9830234692066859, "nll": 3.0275822346617733, "ece10": 0.3634739636502763, "calibration_n": 1088, "confident_wrong": 193 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 1, "accuracy": 0.0025, "wilson95": [ 0.0004414477868016362, 0.01402328507582339 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 700, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35, "brier": 0.8907778495584403, "nll": 1.8791817266199324, "ece10": 0.3140196671152451, "calibration_n": 2000, "confident_wrong": 236 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 23, "accuracy": 0.8214285714285714, "wilson95": [ 0.6440857521065705, 0.9212149805439572 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8214285714285714, "brier": 0.2591123624637336, "nll": 1.5681605612042842, "ece10": 0.11775075148672179, "calibration_n": 28, "confident_wrong": 1, "macro_f1": 0.8184981684981685, "confusion_matrix": { "design": { "design": 5, "inspection": 1, "purchase": 1, "work": 0, "__failed__": 0 }, "inspection": { "design": 2, "inspection": 5, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 6, "work": 1, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 21, "accuracy": 0.75, "wilson95": [ 0.5664432968100104, 0.8732350508070668 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3479823325595138, "nll": 1.7475202474255096, "ece10": 0.14850181370880386, "calibration_n": 28, "confident_wrong": 2, "macro_f1": 0.7476190476190475, "confusion_matrix": { "work": { "work": 4, "purchase": 0, "inspection": 1, "design": 2, "__failed__": 0 }, "purchase": { "work": 1, "purchase": 6, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 6, "design": 1, "__failed__": 0 }, "design": { "work": 0, "purchase": 1, "inspection": 1, "design": 5, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 22, "accuracy": 0.7857142857142857, "wilson95": [ 0.6046141434627812, 0.8978753966710211 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.38761441521858503, "nll": 1.8813196009066435, "ece10": 0.20452878738093075, "calibration_n": 28, "confident_wrong": 2, "macro_f1": 0.7792654713707345, "confusion_matrix": { "design": { "design": 6, "inspection": 1, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 3, "purchase": 3, "work": 1, "__failed__": 0 }, "work": { "design": 0, "inspection": 1, "purchase": 0, "work": 6, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 19, "accuracy": 0.6785714285714286, "wilson95": [ 0.49338843180857617, 0.8206675307750504 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6785714285714286, "brier": 0.4770653050480507, "nll": 2.064507771365165, "ece10": 0.18390876456487334, "calibration_n": 28, "confident_wrong": 2, "macro_f1": 0.6581174136321195, "confusion_matrix": { "work": { "work": 6, "purchase": 0, "inspection": 1, "design": 0, "__failed__": 0 }, "purchase": { "work": 1, "purchase": 2, "inspection": 2, "design": 2, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 5, "design": 2, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 1, "design": 6, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 19, "accuracy": 0.475, "wilson95": [ 0.329354701126789, 0.6250263789330753 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.475, "brier": 0.7937406756771508, "nll": 2.6722758948752015, "ece10": 0.3323206326393803, "calibration_n": 40, "confident_wrong": 6, "macro_f1": 0.45904306220095686, "confusion_matrix": { "review": { "review": 2, "it": 1, "documents": 1, "procurement": 1, "maintenance": 3, "__failed__": 0 }, "it": { "review": 1, "it": 4, "documents": 0, "procurement": 2, "maintenance": 1, "__failed__": 0 }, "documents": { "review": 2, "it": 0, "documents": 2, "procurement": 2, "maintenance": 2, "__failed__": 0 }, "procurement": { "review": 1, "it": 0, "documents": 0, "procurement": 6, "maintenance": 1, "__failed__": 0 }, "maintenance": { "review": 1, "it": 2, "documents": 0, "procurement": 0, "maintenance": 5, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 20, "accuracy": 0.5, "wilson95": [ 0.35199526933465386, 0.6480047306653461 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.7386924394449966, "nll": 2.7069695760687233, "ece10": 0.31326159625486594, "calibration_n": 40, "confident_wrong": 8, "macro_f1": 0.4660593792172739, "confusion_matrix": { "maintenance": { "maintenance": 7, "procurement": 0, "documents": 0, "it": 1, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 2, "procurement": 6, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 2, "procurement": 2, "documents": 2, "it": 0, "review": 2, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 2, "documents": 0, "it": 4, "review": 1, "__failed__": 0 }, "review": { "maintenance": 4, "procurement": 1, "documents": 2, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 18, "accuracy": 0.45, "wilson95": [ 0.3070530781090492, 0.6017090820106794 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.8209927314229286, "nll": 1.758078100900678, "ece10": 0.3497468076921931, "calibration_n": 40, "confident_wrong": 4, "macro_f1": 0.42937638213180007, "confusion_matrix": { "review": { "review": 1, "it": 0, "documents": 0, "procurement": 3, "maintenance": 4, "__failed__": 0 }, "it": { "review": 0, "it": 5, "documents": 0, "procurement": 1, "maintenance": 2, "__failed__": 0 }, "documents": { "review": 3, "it": 1, "documents": 2, "procurement": 1, "maintenance": 1, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 7, "maintenance": 1, "__failed__": 0 }, "maintenance": { "review": 2, "it": 3, "documents": 0, "procurement": 0, "maintenance": 3, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 18, "accuracy": 0.45, "wilson95": [ 0.3070530781090492, 0.6017090820106794 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.8057155256321543, "nll": 1.8709159682106349, "ece10": 0.3289480234972384, "calibration_n": 40, "confident_wrong": 5, "macro_f1": 0.39521531100478463, "confusion_matrix": { "maintenance": { "maintenance": 5, "procurement": 0, "documents": 1, "it": 2, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 1, "procurement": 6, "documents": 0, "it": 1, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 2, "procurement": 2, "documents": 2, "it": 2, "review": 0, "__failed__": 0 }, "it": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 5, "review": 1, "__failed__": 0 }, "review": { "maintenance": 4, "procurement": 3, "documents": 0, "it": 1, "review": 0, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 215, "accuracy": 0.5375, "wilson95": [ 0.4885138311983594, 0.5857727467265683 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 215, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5375, "brier": 0.7805394483362436, "nll": 2.4407878000309036, "ece10": 0.3307273750316715, "calibration_n": 400, "confident_wrong": 95, "macro_f1": 0.43423290131567743, "confusion_matrix": { "sadness": { "sadness": 83, "joy": 12, "love": 7, "anger": 6, "fear": 9, "surprise": 2, "__failed__": 0 }, "joy": { "sadness": 11, "joy": 71, "love": 28, "anger": 3, "fear": 2, "surprise": 3, "__failed__": 0 }, "love": { "sadness": 8, "joy": 12, "love": 14, "anger": 0, "fear": 3, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 26, "joy": 5, "love": 6, "anger": 22, "fear": 2, "surprise": 3, "__failed__": 0 }, "fear": { "sadness": 20, "joy": 1, "love": 1, "anger": 1, "fear": 24, "surprise": 5, "__failed__": 0 }, "surprise": { "sadness": 3, "joy": 3, "love": 1, "anger": 1, "fear": 1, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 196, "accuracy": 0.6533333333333333, "wilson95": [ 0.5978480299293977, 0.7049414588950748 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 196, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6533333333333333, "brier": 0.5758736498553134, "nll": 2.8638247859867607, "ece10": 0.23097385531791198, "calibration_n": 300, "confident_wrong": 46 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 322, "accuracy": 0.805, "wilson95": [ 0.7633481418022936, 0.8408493586537855 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 322, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.805, "brier": 0.33447850305000026, "nll": 1.8672008853622442, "ece10": 0.1530082499999999, "calibration_n": 400, "confident_wrong": 54, "macro_f1": 0.7667394359879183, "confusion_matrix": { "false": { "false": 242, "true": 67, "__failed__": 0 }, "true": { "false": 11, "true": 80, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 375, "accuracy": 0.9375, "wilson95": [ 0.9093658838696931, 0.9573108585877975 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 375, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.09794457239625429, "nll": 0.2957736053154989, "ece10": 0.035673789381502924, "calibration_n": 400, "confident_wrong": 12, "macro_f1": 0.9311456392013824, "confusion_matrix": { "world": { "world": 97, "sports": 1, "business": 4, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 3, "sports": 120, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 0, "sports": 0, "business": 65, "sci_tech": 7, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 7, "sci_tech": 93, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 397, "accuracy": 0.9925, "wilson95": [ 0.9781842792759678, 0.9974461108047501 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 397, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 7, "head_accuracy": 0.9925, "brier": 0.013797740399999996, "nll": 0.024327668995275716, "ece10": 0.006269500000000003, "calibration_n": 400, "confident_wrong": 3, "macro_f1": 0.9920906412512606, "confusion_matrix": { "true": { "true": 153, "false": 0, "__failed__": 0 }, "false": { "true": 3, "false": 244, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 168, "accuracy": 0.56, "wilson95": [ 0.503422038437878, 0.6150608050151765 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 168, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.56, "brier": 0.6993656153233749, "nll": 3.2961548530932534, "ece10": 0.3068222284392712, "calibration_n": 300, "confident_wrong": 55 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 399, "accuracy": 0.9975, "wilson95": [ 0.9859767149241766, 0.9995585522131983 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 399, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 9, "head_accuracy": 0.9975, "brier": 0.004961685100000001, "nll": 0.012046416465853457, "ece10": 0.0026275000000000005, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.9974917069561237, "confusion_matrix": { "false": { "false": 188, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 211, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 214, "accuracy": 0.535, "wilson95": [ 0.486020056782804, 0.5833140826137954 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 214, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.535, "brier": 0.7372244774000003, "nll": 1.220736319449079, "ece10": 0.37267749999999994, "calibration_n": 400, "confident_wrong": 101, "macro_f1": 0.41360068097985436, "confusion_matrix": { "false": { "false": 198, "true": 2, "__failed__": 0 }, "true": { "false": 184, "true": 16, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 47, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47, "brier": 0.8355320762000004, "nll": 1.4480512864620032, "ece10": 0.39000900000000005, "calibration_n": 100, "confident_wrong": 25, "macro_f1": 0.396011396011396, "confusion_matrix": { "yes": { "yes": 6, "no": 48, "__failed__": 0 }, "no": { "yes": 5, "no": 41, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 38, "accuracy": 0.38, "wilson95": [ 0.29097599252478734, 0.47790244704488916 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.38, "brier": 1.1693699418749735, "nll": 10.990488792632721, "ece10": 0.5584177376802317, "calibration_n": 100, "confident_wrong": 45, "macro_f1": 0.38794287766327235, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 2, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 1, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 4, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 1, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 1, "dispute_charge": 0, "loan_payment": 0, "mortgage": 4, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 2, "fee_refund": 0, "direct_deposit": 0, "close_account": 1, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 1, "card_lost": 1, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "statement": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 3, "mortgage": 0, "wire": 3, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 1, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 3, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 4, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 3, "statement": 0, "fraud_alert": 2, "branch_appointment": 2, "fee_refund": 1, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 1, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 4, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 1, "new_account": 4, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 3, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 0, "card_lost": 2, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 38, "accuracy": 0.38, "wilson95": [ 0.29097599252478734, 0.47790244704488916 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 124, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.62, "brier": 0.5631609276042, "nll": 1.199613932857515, "ece10": 0.2309410390526967, "calibration_n": 200, "confident_wrong": 31 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 18, "accuracy": 0.18, "wilson95": [ 0.1170022549894635, 0.2666735838630073 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 94, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47, "brier": 0.9314747590575909, "nll": 6.802259462121916, "ece10": 0.39990074298934797, "calibration_n": 200, "confident_wrong": 60 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 69, "accuracy": 0.69, "wilson95": [ 0.5937394428051057, 0.7722030278762395 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.69, "brier": 0.4994395920652861, "nll": 2.7032950295228773, "ece10": 0.19050698877482194, "calibration_n": 100, "confident_wrong": 14, "macro_f1": 0.6791189699096086, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 4, "receipt": 12, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 1, "receipt": 1, "contract": 3, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 3, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 9, "clinical_note": 1, "syllabus": 2, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 7, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 1, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 2, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 1, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 3, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 4, "accuracy": 0.04, "wilson95": [ 0.01566330399154762, 0.0983707143588792 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 173, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4325, "brier": 0.9659841284172879, "nll": 5.9054983526932645, "ece10": 0.44286332237026566, "calibration_n": 400, "confident_wrong": 136 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 46, "accuracy": 0.46, "wilson95": [ 0.36560808309024106, 0.5573513967663177 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 46, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.46, "brier": 0.8491424182621978, "nll": 2.355241366397626, "ece10": 0.3663252029947138, "calibration_n": 100, "confident_wrong": 20, "macro_f1": 0.43640971798866535, "confusion_matrix": { "politics": { "politics": 0, "business": 2, "technology": 4, "science": 5, "health": 2, "sports": 0, "entertainment": 0, "world": 0, "culture": 1, "weather": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "technology": { "politics": 0, "business": 4, "technology": 5, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 1, "technology": 1, "science": 7, "health": 0, "sports": 1, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "health": { "politics": 0, "business": 3, "technology": 1, "science": 1, "health": 5, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 1, "science": 1, "health": 0, "sports": 6, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 0, "business": 0, "technology": 2, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 1, "weather": 1, "__failed__": 0 }, "world": { "politics": 0, "business": 6, "technology": 1, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 3, "science": 2, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 3, "weather": 1, "__failed__": 0 }, "weather": { "politics": 0, "business": 2, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 6, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 31, "accuracy": 0.31, "wilson95": [ 0.22779697212376038, 0.40626055719489407 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 31, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.31, "brier": 1.056370472123276, "nll": 3.2597582968163663, "ece10": 0.3729065574120317, "calibration_n": 100, "confident_wrong": 18, "macro_f1": 0.27816856877732393, "confusion_matrix": { "biology": { "biology": 3, "medicine": 3, "chemistry": 2, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 9, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 1, "chemistry": 5, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 1, "medicine": 4, "chemistry": 4, "physics": 5, "computer_science": 0, "mathematics": 1, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 1, "medicine": 4, "chemistry": 5, "physics": 0, "computer_science": 1, "mathematics": 2, "earth_science": 0, "engineering": 1, "economics": 1, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 1, "chemistry": 1, "physics": 0, "computer_science": 1, "mathematics": 2, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 1, "medicine": 4, "chemistry": 2, "physics": 1, "computer_science": 2, "mathematics": 1, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 3, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 0, "engineering": 3, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 2, "mathematics": 1, "earth_science": 0, "engineering": 0, "economics": 3, "other": 0, "__failed__": 0 }, "other": { "biology": 1, "medicine": 2, "chemistry": 1, "physics": 4, "computer_science": 1, "mathematics": 1, "earth_science": 0, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 1, "accuracy": 0.01, "wilson95": [ 0.0017674320641406505, 0.054486196178705315 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 50, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.25, "brier": 0.9630799514260927, "nll": 2.997016365571891, "ece10": 0.4664365056666253, "calibration_n": 100, "confident_wrong": 40 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 1, "accuracy": 0.01, "wilson95": [ 0.0017674320641406505, 0.054486196178705315 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 68, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.34, "brier": 0.5546019635307293, "nll": 1.2263454091221155, "ece10": 0.22578334975452358, "calibration_n": 100, "confident_wrong": 17 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 63, "accuracy": 0.63, "wilson95": [ 0.5322052958094291, 0.7181763946567546 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 63, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.63, "brier": 0.675614633378768, "nll": 1.8637636023393103, "ece10": 0.3245793877304839, "calibration_n": 100, "confident_wrong": 29, "macro_f1": 0.5931236688518241, "confusion_matrix": { "negative": { "negative": 36, "neutral": 0, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 25, "neutral": 7, "positive": 1, "__failed__": 0 }, "positive": { "negative": 6, "neutral": 5, "positive": 20, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 68, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.34, "brier": 0.4536862882000003, "nll": 1.0329306553757862, "ece10": 0.164517, "calibration_n": 100, "confident_wrong": 9 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 16, "accuracy": 0.16, "wilson95": [ 0.10095288488804782, 0.24420269389270244 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 170, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5666666666666667, "brier": 0.6842611254898133, "nll": 1.8701644417548433, "ece10": 0.294660805666639, "calibration_n": 300, "confident_wrong": 60 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 27, "accuracy": 0.27, "wilson95": [ 0.1926958414957634, 0.36432116767944994 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.27, "brier": 1.3073516619953125, "nll": 13.25253903923764, "ece10": 0.634633404291925, "calibration_n": 100, "confident_wrong": 51, "macro_f1": 0.2766128712557284, "confusion_matrix": { "order_status": { "order_status": 1, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 1, "password_reset": 1, "shipping_delay": 0, "missing_item": 0, "damaged_item": 1, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 1, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 1, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 1, "refund_request": 0, "cancel_subscription": 1, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 2, "refund_request": 0, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 1, "refund_request": 0, "cancel_subscription": 2, "update_payment": 2, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 1, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 4, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 4, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 2, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 3, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 1, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 2, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 1, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 2, "refund_request": 1, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "other": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 24, "accuracy": 0.24, "wilson95": [ 0.16691325556948772, 0.33232336349814473 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 24, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.24, "brier": 1.3806009140440019, "nll": 15.015515935872784, "ece10": 0.6475090367100533, "calibration_n": 100, "confident_wrong": 56, "macro_f1": 0.21647852147852145, "confusion_matrix": { "billing": { "billing": 4, "account_access": 0, "shipping": 1, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 2, "shipping": 2, "returns": 4, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 1, "account_access": 0, "shipping": 4, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 1, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 3, "account_access": 1, "shipping": 2, "returns": 3, "technical_outage": 1, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 3, "integrations": 0, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 3, "account_access": 0, "shipping": 1, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "compliance": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "pricing": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 2, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 1, "account_access": 0, "shipping": 1, "returns": 4, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 1, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "other": { "billing": 3, "account_access": 0, "shipping": 3, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 114, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.38, "brier": 0.9574333469844523, "nll": 5.72222347007492, "ece10": 0.3947723931226448, "calibration_n": 300, "confident_wrong": 78 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 29, "accuracy": 0.29, "wilson95": [ 0.2101483574912227, 0.3853889117557111 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 29, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.29, "brier": 1.3124423311631883, "nll": 11.234276328527177, "ece10": 0.6149805415919332, "calibration_n": 100, "confident_wrong": 48, "macro_f1": 0.26633730180241805, "confusion_matrix": { "new_booking": { "new_booking": 4, "change_flight": 0, "cancel_trip": 1, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 7, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 2, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 2, "change_flight": 0, "cancel_trip": 2, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 2, "change_flight": 1, "cancel_trip": 0, "seat": 3, "baggage": 0, "refund": 1, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 7, "refund": 2, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 5, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 4, "change_flight": 0, "cancel_trip": 1, "seat": 1, "baggage": 0, "refund": 1, "hotel": 2, "car": 0, "visa_docs": 1, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "car": { "new_booking": 3, "change_flight": 0, "cancel_trip": 1, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 3, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 1, "hotel": 2, "car": 0, "visa_docs": 1, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 3, "change_flight": 0, "cancel_trip": 2, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 4, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 4, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 1, "disruption": 1, "other": 0, "__failed__": 0 }, "other": { "new_booking": 4, "change_flight": 0, "cancel_trip": 1, "seat": 1, "baggage": 1, "refund": 0, "hotel": 2, "car": 1, "visa_docs": 0, "loyalty": 1, "disruption": 1, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 94, "accuracy": 0.94, "wilson95": [ 0.8752318455410406, 0.9722138760368118 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 94, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.94, "brier": 0.09286916306631164, "nll": 0.2775916514653607, "ece10": 0.025779198677662, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.939271525304066, "confusion_matrix": { "business": { "business": 24, "science and technology": 1, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 2, "science and technology": 21, "sports": 1, "world": 1, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 1, "science and technology": 0, "sports": 0, "world": 24, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 35, "accuracy": 0.35, "wilson95": [ 0.26364248243097527, 0.44745556703112027 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 100, "head_accuracy": 0.35, "brier": 1.000290285440978, "nll": 6.089008933875701, "ece10": 0.42973119898976037, "calibration_n": 100, "confident_wrong": 22, "macro_f1": 0.2783068783068783, "confusion_matrix": { "refund not showing up": { "refund not showing up": 1, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 1, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 1, "age limits": 1, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 1, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 1, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 1, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 1, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 2, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 2, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 1, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 1, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 1, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 1, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 41, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.41, "brier": 0.987805373511935, "nll": 3.5699561891919758, "ece10": 0.4558831030934011, "calibration_n": 100, "confident_wrong": 31, "macro_f1": 0.4176246941464332, "confusion_matrix": { "anger": { "anger": 7, "fear": 1, "joy": 2, "love": 1, "sadness": 6, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 0, "fear": 6, "joy": 0, "love": 0, "sadness": 10, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 6, "love": 6, "sadness": 4, "surprise": 1, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 2, "love": 8, "sadness": 6, "surprise": 1, "__failed__": 0 }, "sadness": { "anger": 3, "fear": 1, "joy": 2, "love": 0, "sadness": 10, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 0, "joy": 4, "love": 3, "sadness": 4, "surprise": 4, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1815, "accuracy": 0.6156716417910447, "wilson95": [ 0.5979724741601596, 0.6330697445748766 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1815, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6156716417910447, "brier": 0.5480326307933283, "nll": 1.61678346029386, "ece10": 0.17359216276439293, "calibration_n": 2948, "confident_wrong": 217, "macro_f1": 0.6130195944163183, "confusion_matrix": { "alarm": { "alarm": 94, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 6, "audio": 34, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 5, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 10, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "calendar": { "alarm": 23, "audio": 0, "calendar": 281, "cooking": 5, "datetime": 25, "email": 1, "general": 14, "iot": 11, "lists": 1, "music": 4, "news": 2, "play": 1, "qa": 21, "recommendation": 1, "social": 1, "takeaway": 2, "transport": 3, "weather": 4, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 68, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 74, "email": 0, "general": 1, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 13, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 3, "weather": 7, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 3, "email": 211, "general": 17, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 28, "recommendation": 0, "social": 2, "takeaway": 1, "transport": 1, "weather": 1, "__failed__": 0 }, "general": { "alarm": 5, "audio": 1, "calendar": 8, "cooking": 4, "datetime": 3, "email": 0, "general": 34, "iot": 8, "lists": 1, "music": 4, "news": 1, "play": 15, "qa": 83, "recommendation": 9, "social": 1, "takeaway": 3, "transport": 2, "weather": 5, "__failed__": 0 }, "iot": { "alarm": 31, "audio": 1, "calendar": 0, "cooking": 21, "datetime": 0, "email": 0, "general": 2, "iot": 145, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 11, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 1, "datetime": 0, "email": 0, "general": 1, "iot": 4, "lists": 102, "music": 6, "news": 0, "play": 1, "qa": 17, "recommendation": 0, "social": 0, "takeaway": 7, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 9, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 2, "music": 53, "news": 0, "play": 3, "qa": 9, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 1, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 1, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 76, "play": 1, "qa": 31, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 4, "weather": 3, "__failed__": 0 }, "play": { "alarm": 2, "audio": 119, "calendar": 0, "cooking": 1, "datetime": 1, "email": 1, "general": 17, "iot": 15, "lists": 2, "music": 44, "news": 1, "play": 141, "qa": 37, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 2, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 1, "datetime": 9, "email": 0, "general": 43, "iot": 11, "lists": 1, "music": 4, "news": 1, "play": 7, "qa": 178, "recommendation": 11, "social": 1, "takeaway": 2, "transport": 5, "weather": 11, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 8, "cooking": 3, "datetime": 2, "email": 0, "general": 12, "iot": 1, "lists": 1, "music": 1, "news": 3, "play": 3, "qa": 16, "recommendation": 33, "social": 0, "takeaway": 1, "transport": 9, "weather": 1, "__failed__": 0 }, "social": { "alarm": 0, "audio": 2, "calendar": 4, "cooking": 1, "datetime": 0, "email": 1, "general": 8, "iot": 8, "lists": 0, "music": 1, "news": 3, "play": 0, "qa": 11, "recommendation": 1, "social": 60, "takeaway": 3, "transport": 1, "weather": 1, "__failed__": 0 }, "takeaway": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 11, "datetime": 2, "email": 0, "general": 1, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 10, "recommendation": 0, "social": 0, "takeaway": 27, "transport": 2, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 4, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 9, "email": 0, "general": 0, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 11, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 91, "weather": 2, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 0, "general": 3, "iot": 21, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 12, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 113, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1511, "accuracy": 0.512550881953867, "wilson95": [ 0.494502922473108, 0.5305661745743885 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1511, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.512550881953867, "brier": 0.7210396572047719, "nll": 2.173662923509086, "ece10": 0.25267408202885894, "calibration_n": 2948, "confident_wrong": 328, "macro_f1": 0.5253281273250408, "confusion_matrix": { "alarm": { "alarm": 92, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 3, "audio": 26, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 16, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 12, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 65, "audio": 0, "calendar": 161, "cooking": 9, "datetime": 16, "email": 2, "general": 36, "iot": 28, "lists": 1, "music": 8, "news": 8, "play": 2, "qa": 47, "recommendation": 3, "social": 1, "takeaway": 1, "transport": 3, "weather": 9, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 64, "datetime": 0, "email": 0, "general": 0, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 3, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 27, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 46, "email": 0, "general": 1, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 10, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 15, "__failed__": 0 }, "email": { "alarm": 1, "audio": 0, "calendar": 4, "cooking": 2, "datetime": 1, "email": 164, "general": 24, "iot": 19, "lists": 15, "music": 0, "news": 1, "play": 0, "qa": 30, "recommendation": 1, "social": 4, "takeaway": 0, "transport": 1, "weather": 2, "__failed__": 0 }, "general": { "alarm": 6, "audio": 1, "calendar": 5, "cooking": 9, "datetime": 0, "email": 0, "general": 35, "iot": 11, "lists": 1, "music": 5, "news": 0, "play": 7, "qa": 90, "recommendation": 5, "social": 0, "takeaway": 2, "transport": 5, "weather": 5, "__failed__": 0 }, "iot": { "alarm": 17, "audio": 1, "calendar": 0, "cooking": 20, "datetime": 0, "email": 0, "general": 5, "iot": 145, "lists": 1, "music": 1, "news": 0, "play": 3, "qa": 16, "recommendation": 0, "social": 0, "takeaway": 3, "transport": 0, "weather": 2, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 5, "calendar": 1, "cooking": 9, "datetime": 0, "email": 0, "general": 9, "iot": 9, "lists": 68, "music": 7, "news": 0, "play": 3, "qa": 25, "recommendation": 1, "social": 1, "takeaway": 4, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 6, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 1, "lists": 0, "music": 62, "news": 0, "play": 6, "qa": 5, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 2, "datetime": 1, "email": 0, "general": 13, "iot": 0, "lists": 0, "music": 0, "news": 48, "play": 1, "qa": 51, "recommendation": 2, "social": 1, "takeaway": 0, "transport": 1, "weather": 3, "__failed__": 0 }, "play": { "alarm": 3, "audio": 65, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 20, "iot": 29, "lists": 3, "music": 77, "news": 2, "play": 127, "qa": 54, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "qa": { "alarm": 1, "audio": 0, "calendar": 6, "cooking": 1, "datetime": 9, "email": 0, "general": 45, "iot": 22, "lists": 1, "music": 7, "news": 3, "play": 3, "qa": 169, "recommendation": 9, "social": 0, "takeaway": 0, "transport": 3, "weather": 8, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 3, "datetime": 2, "email": 0, "general": 13, "iot": 0, "lists": 0, "music": 4, "news": 2, "play": 6, "qa": 16, "recommendation": 34, "social": 0, "takeaway": 4, "transport": 6, "weather": 2, "__failed__": 0 }, "social": { "alarm": 0, "audio": 2, "calendar": 3, "cooking": 2, "datetime": 0, "email": 0, "general": 12, "iot": 15, "lists": 0, "music": 1, "news": 2, "play": 0, "qa": 15, "recommendation": 1, "social": 49, "takeaway": 2, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 10, "datetime": 2, "email": 0, "general": 3, "iot": 8, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 6, "recommendation": 1, "social": 0, "takeaway": 25, "transport": 1, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 5, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 1, "general": 0, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 21, "recommendation": 1, "social": 0, "takeaway": 2, "transport": 87, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 4, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 0, "email": 0, "general": 5, "iot": 22, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 8, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 109, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 696, "accuracy": 0.5933503836317136, "wilson95": [ 0.5649796692146967, 0.6211116673528123 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 696, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5933503836317136, "cluster_bootstrap95": [ 0.5615177331629523, 0.6219334448782302 ], "independent_source_groups": 47, "brier": 0.6160334702788035, "nll": 1.2015411333167536, "ece10": 0.21590891215293395, "calibration_n": 1173, "confident_wrong": 138, "macro_f1": 0.5337026667663335, "confusion_matrix": { "negative": { "negative": 75, "neutral": 102, "positive": 5, "__failed__": 0 }, "neutral": { "negative": 87, "neutral": 462, "positive": 41, "__failed__": 0 }, "positive": { "negative": 32, "neutral": 210, "positive": 159, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "laya": { "label": "Laya", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 183, "accuracy": 0.915, "wilson95": [ 0.8681041292843443, 0.9462542498225244 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 183, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.915, "brier": 0.13194010331061515, "nll": 0.24573290266841546, "ece10": 0.02029384224050342, "calibration_n": 200, "confident_wrong": 3, "macro_f1": 0.916119651657603, "confusion_matrix": { "world": { "world": 48, "sports": 0, "business": 0, "sci_tech": 2, "__failed__": 0 }, "sports": { "world": 0, "sports": 45, "business": 2, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 1, "sports": 0, "business": 38, "sci_tech": 12, "__failed__": 0 }, "sci_tech": { "world": 0, "sports": 0, "business": 0, "sci_tech": 52, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 121, "accuracy": 0.605, "wilson95": [ 0.5358834915928536, 0.6701589898679203 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 121, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.605, "brier": 0.6856944849608406, "nll": 2.2821011761939456, "ece10": 0.3099720055463556, "calibration_n": 200, "confident_wrong": 47, "macro_f1": 0.4465214798331259, "confusion_matrix": { "sadness": { "sadness": 42, "joy": 3, "love": 2, "anger": 7, "fear": 3, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 15, "joy": 52, "love": 8, "anger": 2, "fear": 2, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 1, "joy": 5, "love": 5, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 9, "joy": 3, "love": 1, "anger": 14, "fear": 1, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 6, "joy": 2, "love": 1, "anger": 2, "fear": 8, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 4, "love": 0, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 200, "accuracy": 1.0, "wilson95": [ 0.9811546736227335, 1.0 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 200, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 39, "head_accuracy": 1.0, "brier": 0.0045690333, "nll": 0.014259171864394217, "ece10": 0.012781499999999998, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 109, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 91, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": 0.6, "development": { "n": 99, "correct": 99, "wilson95": [ 0.9626467879321726, 1.0 ] }, "test_coverage": 0.995, "accepted_n": 199, "accepted_correct": 199, "accepted_accuracy": 1.0, "accepted_wilson95": [ 0.9810617669433649, 0.9999999999999999 ], "errors_per_1000_all_test_cases": 0.0, "note": "Selected on development only, applied unchanged to test. An empirical pilot estimate, not a production guarantee." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 200, "accuracy": 1.0, "wilson95": [ 0.9811546736227335, 1.0 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 200, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 56, "head_accuracy": 1.0, "brier": 0.004736576099999998, "nll": 0.013025312272275673, "ece10": 0.011418499999999998, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "true": { "true": 77, "false": 0, "__failed__": 0 }, "false": { "true": 0, "false": 123, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": 0.8, "development": { "n": 97, "correct": 97, "wilson95": [ 0.9619059574740523, 1.0 ] }, "test_coverage": 0.99, "accepted_n": 198, "accepted_correct": 198, "accepted_accuracy": 1.0, "accepted_wilson95": [ 0.9809679396733518, 1.0 ], "errors_per_1000_all_test_cases": 0.0, "note": "Selected on development only, applied unchanged to test. An empirical pilot estimate, not a production guarantee." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.5897929703548419, "nll": 1.207915087917719, "ece10": 0.22194239176825722, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.6742857142857142, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 1, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 1, "documents": 0, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 3, "accuracy": 0.25, "wilson95": [ 0.08894166839405471, 0.5323053349335657 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 3, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 12, "head_accuracy": 0.25, "brier": 0.8150337618724284, "nll": 1.655804061503032, "ece10": 0.30319327777901617, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.08, "confusion_matrix": { "maintenance": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 0, "review": 3, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.44460697965412255, "nll": 0.7976345392881146, "ece10": 0.27338630372894634, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.6666666666666666, "confusion_matrix": { "design": { "design": 2, "inspection": 0, "purchase": 1, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 2, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 1, "purchase": 2, "work": 0, "__failed__": 0 }, "work": { "design": 1, "inspection": 0, "purchase": 0, "work": 2, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 4, "accuracy": 0.3333333333333333, "wilson95": [ 0.13812009109121315, 0.6093779111272004 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 4, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3333333333333333, "brier": 0.6967790199942386, "nll": 1.2660160823628868, "ece10": 0.2196728656198953, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.26136363636363635, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 2, "inspection": 0, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 1, "work": 1, "__failed__": 0 }, "work": { "design": 2, "inspection": 1, "purchase": 0, "work": 0, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 97, "accuracy": 0.97, "wilson95": [ 0.9154806357094724, 0.9897454759759611 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 97, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.97, "brier": 0.09941611358150544, "nll": 0.2690843957108368, "ece10": 0.14989857344790572, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.9699754901960784, "confusion_matrix": { "label_000": { "label_000": 23, "label_001": 2, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 24, "label_002": 1, "label_003": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 40, "accuracy": 0.4, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.9615488315974045, "nll": 2.426368351769061, "ece10": 0.4130517286398314, "calibration_n": 100, "confident_wrong": 22, "macro_f1": 0.37363383532848143, "confusion_matrix": { "label_000": { "label_000": 7, "label_001": 3, "label_002": 2, "label_003": 0, "label_004": 5, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 9, "label_002": 2, "label_003": 0, "label_004": 5, "label_005": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 13, "label_003": 1, "label_004": 3, "label_005": 0, "__failed__": 0 }, "label_003": { "label_000": 1, "label_001": 0, "label_002": 11, "label_003": 2, "label_004": 3, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 2, "label_001": 3, "label_002": 4, "label_003": 0, "label_004": 7, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 0, "label_002": 8, "label_003": 0, "label_004": 5, "label_005": 2, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 2, "accuracy": 0.02, "wilson95": [ 0.00550196755016235, 0.07001179072854388 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 2, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 100, "head_accuracy": 0.02, "brier": 1.0015547640811722, "nll": 4.883051781835888, "ece10": 0.0864402233965475, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.0006858710562414266, "confusion_matrix": { "label_000": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 1, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 2, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 46, "accuracy": 0.9583333333333334, "wilson95": [ 0.8602434412954456, 0.9884981065106133 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 46, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9583333333333334, "brier": 0.06654678800173876, "nll": 0.13214228088411065, "ece10": 0.05609760344416349, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 37, "accuracy": 0.3333333333333333, "wilson95": [ 0.2525113750360455, 0.4253053265284187 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 57, "head_accuracy": 0.3333333333333333, "brier": 0.7941087132668547, "nll": 1.3668553779585595, "ece10": 0.2062548912587561, "calibration_n": 111, "confident_wrong": 2 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 52, "accuracy": 0.7222222222222222, "wilson95": [ 0.6095356707476457, 0.8123971414317063 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 52, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "cluster_bootstrap95": [ 0.5972222222222222, 0.8333333333333334 ], "independent_source_groups": 36, "brier": 0.43928761079899853, "nll": 0.7760650019238844, "ece10": 0.07243224154419742, "calibration_n": 72, "confident_wrong": 2 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 11, "accuracy": 0.5789473684210527, "wilson95": [ 0.3627590353425504, 0.7685810871754576 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5789473684210527, "brier": 0.5042901368421053, "nll": 0.7032663649793017, "ece10": 0.20017894736842107, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.5476190476190476, "confusion_matrix": { "false": { "false": 8, "true": 3, "__failed__": 0 }, "true": { "false": 5, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.4041479925, "nll": 0.5960863784035438, "ece10": 0.17824999999999996, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.7333333333333334, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 11, "accuracy": 0.5, "wilson95": [ 0.3072210627372502, 0.6927789372627497 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6864926454545454, "nll": 0.9496527763707459, "ece10": 0.3342818181818182, "calibration_n": 22, "confident_wrong": 1, "macro_f1": 0.39950372208436724, "confusion_matrix": { "false": { "false": 10, "true": 0, "__failed__": 0 }, "true": { "false": 11, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.136071523118637, "nll": 0.3174431535701369, "ece10": 0.16780190876676906, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9415584415584415, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 3, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 7, "accuracy": 0.3888888888888889, "wilson95": [ 0.2030524657216362, 0.613809583977621 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3888888888888889, "brier": 0.5542835407494543, "nll": 1.0326581920068096, "ece10": 0.33643387197956093, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.37714285714285717, "confusion_matrix": { "0": { "0": 1, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 3, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 3, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 14, "accuracy": 0.7368421052631579, "wilson95": [ 0.5120844910218489, 0.8819358765321748 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7368421052631579, "brier": 0.40406618669643496, "nll": 0.8247138748937143, "ece10": 0.21755235470915513, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.7411688311688313, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 1, "roads": 3, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 2, "roads": 0, "noise": 2, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 1, "permits": 3, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 1, "noise": 0, "permits": 0, "utilities": 2, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 7, "accuracy": 0.4117647058823529, "wilson95": [ 0.2161105679021481, 0.6399455735700021 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4117647058823529, "brier": 0.7069199699490768, "nll": 1.5171384089033433, "ece10": 0.2961722131036633, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.42023809523809524, "confusion_matrix": { "bug": { "bug": 1, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 1, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 1, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 1, "refactor": 1, "nit": 1, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 1, "refactor": 1, "nit": 2, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 12, "accuracy": 0.8, "wilson95": [ 0.5481455128483065, 0.9295245065301844 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.3546774567140924, "nll": 0.7516469699505284, "ece10": 0.2757433288190999, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7900793650793649, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 1, "fix": 1, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 1, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 2, "chore": 0, "__failed__": 0 }, "chore": { "feature": 1, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 3, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.28940643058823534, "nll": 0.43951213079667445, "ece10": 0.1220235294117647, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7638888888888888, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.5761597574999999, "nll": 0.7643705487774562, "ece10": 0.3211125, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.4181818181818182, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 7, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.3985015374258378, "nll": 0.7908423308876659, "ece10": 0.11355782181163375, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5887955182072829, "confusion_matrix": { "harassment": { "harassment": 2, "threat": 0, "self_harm": 0, "spam": 0, "none": 3, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 1, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 6, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 14, "accuracy": 0.6363636363636364, "wilson95": [ 0.42951708090131013, 0.8026679532963387 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6363636363636364, "brier": 0.5196954509172491, "nll": 0.9582494970865412, "ece10": 0.1739078653608059, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.6203174603174603, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 1, "opinion": 4, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 1, "opinion": 0, "press_release": 3, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 3, "opinion": 0, "press_release": 0, "analysis": 1, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 3, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 1, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.16968303118432423, "nll": 0.35194639258903976, "ece10": 0.09558323478253822, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8755555555555556, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 1, "termination": 2, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 1, "force_majeure": 2, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.09309459682477624, "nll": 0.2560886404130298, "ece10": 0.2037347041633371, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.704619522352941, "nll": 1.0875159125231002, "ece10": 0.3655294117647059, "calibration_n": 17, "confident_wrong": 4, "macro_f1": 0.4711111111111111, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 7, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 11, "accuracy": 0.7333333333333333, "wilson95": [ 0.4804956594401943, 0.8910254667430761 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7333333333333333, "brier": 0.3753140086515528, "nll": 0.7147451103202105, "ece10": 0.1463910190217102, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7576190476190476, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "purchase_order": 0, "receipt": 2, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 1, "purchase_order": 0, "receipt": 1, "contract": 2, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 1, "contract": 0, "resume": 2, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 11, "accuracy": 0.7333333333333333, "wilson95": [ 0.4804956594401943, 0.8910254667430761 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7333333333333333, "brier": 0.48807040870544344, "nll": 1.2838979227392806, "ece10": 0.20780774723675466, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.6647619047619047, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 0, "complaint": 0, "thanks": 1, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.31939587884564813, "nll": 0.6703929910790981, "ece10": 0.20788746625665244, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.6395959595959596, "confusion_matrix": { "meals": { "meals": 1, "travel": 1, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 2, "travel": 0, "office_supplies": 1, "software": 0, "client_entertainment": 0, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 9, "accuracy": 0.5, "wilson95": [ 0.29031021505169974, 0.7096897849483003 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6516272200000001, "nll": 0.9298998278167073, "ece10": 0.38374444444444444, "calibration_n": 18, "confident_wrong": 2, "macro_f1": 0.3333333333333333, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 9, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 3, "accuracy": 0.1875, "wilson95": [ 0.06591599071428142, 0.4300888096197414 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 3, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.1875, "brier": 0.9680085227457385, "nll": 1.8686614474125331, "ece10": 0.5212147074981471, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.1380952380952381, "confusion_matrix": { "0": { "0": 0, "1": 2, "2": 2, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 9, "accuracy": 0.5, "wilson95": [ 0.29031021505169974, 0.7096897849483003 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.4925350078910319, "nll": 0.7764604595651807, "ece10": 0.27743494002190494, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.3695652173913044, "confusion_matrix": { "0": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 4, "2": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 15, "accuracy": 0.75, "wilson95": [ 0.531299122381256, 0.8881382985923343 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.4204696960727919, "nll": 0.8689940471269472, "ece10": 0.23307019402960194, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7166666666666667, "confusion_matrix": { "cheating": { "cheating": 3, "harassment": 1, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 1, "harassment": 0, "boosting": 1, "smurfing": 1, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 5, "accuracy": 0.23809523809523808, "wilson95": [ 0.10628013022636468, 0.4509117368295075 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.23809523809523808, "brier": 0.849588673734192, "nll": 1.675419144220169, "ece10": 0.21196082490234947, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.1431578947368421, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 1, "none": 3, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 3, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 1, "none": 3, "__failed__": 0 }, "punctuation": { "spelling": 1, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 2, "none": 2, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 0, "punctuation": 1, "none": 3, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.4918805825, "nll": 0.6742000368949612, "ece10": 0.1893, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5151515151515151, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 7, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 20, "accuracy": 0.8695652173913043, "wilson95": [ 0.6787251772990404, 0.9546234090636876 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8695652173913043, "brier": 0.45762683490383405, "nll": 0.9344392774699815, "ece10": 0.4230329407942425, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 0.8658730158730158, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 3, "handyman": 0, "appliance_repair": 1, "__failed__": 0 }, "handyman": { "plumbing": 1, "electrical": 0, "hvac": 0, "handyman": 4, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 1, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 3, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 3, "accuracy": 0.21428571428571427, "wilson95": [ 0.07571386712693184, 0.47589230586600295 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 3, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.21428571428571427, "brier": 0.7627399361462872, "nll": 1.3724527411465208, "ece10": 0.22903974164768312, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.15272727272727274, "confusion_matrix": { "0": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 3, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 1, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 1, "3": 1, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 6, "accuracy": 0.375, "wilson95": [ 0.18481232558863633, 0.6135895945449727 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.375, "brier": 0.7244641768989665, "nll": 1.4555628915446996, "ece10": 0.17917290024579147, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.36964285714285716, "confusion_matrix": { "0": { "0": 1, "1": 2, "2": 1, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 4, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 7, "accuracy": 0.4375, "wilson95": [ 0.23098652405492354, 0.6682144360118811 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4375, "brier": 0.766849576978182, "nll": 1.5221204796032726, "ece10": 0.13631154822074049, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.2902097902097902, "confusion_matrix": { "0": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.7118912470588237, "nll": 0.9895444284970242, "ece10": 0.37534117647058823, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.32, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 9, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.34727502153482726, "nll": 0.6521565188229842, "ece10": 0.121954070617776, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.779047619047619, "confusion_matrix": { "politics": { "politics": 3, "business": 1, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 2, "technology": 2, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 1, "sports": 0, "culture": 2, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.33341346255172816, "nll": 0.6705155407825396, "ece10": 0.19666943232363548, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8414285714285714, "confusion_matrix": { "payments": { "payments": 3, "web": 1, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 1, "infra": 3, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 1, "web": 0, "infra": 0, "data": 2, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.42892631764705885, "nll": 0.9365571050462973, "ece10": 0.20254117647058822, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.6357142857142857, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 5, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.3441171550000001, "nll": 0.49339436294361094, "ece10": 0.15158750000000004, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6536796536796536, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 6, "accuracy": 0.375, "wilson95": [ 0.18481232558863633, 0.6135895945449727 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.375, "brier": 0.6905764510025323, "nll": 1.3479289391634808, "ece10": 0.3575723498420985, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.27499999999999997, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "2": { "0": 2, "1": 4, "2": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 9, "accuracy": 0.5, "wilson95": [ 0.29031021505169974, 0.7096897849483003 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 18, "head_accuracy": 0.5, "brier": 0.5951090562297033, "nll": 1.432698576423843, "ece10": 0.1615322930326007, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.49671717171717167, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 4, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 1, "french": 0, "mexican": 2, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 1, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 0, "indian": 2, "thai": 0, "middle_eastern": 1, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 7, "accuracy": 0.4117647058823529, "wilson95": [ 0.2161105679021481, 0.6399455735700021 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4117647058823529, "brier": 0.7083909670588235, "nll": 0.9605424177153356, "ece10": 0.3916470588235294, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.356060606060606, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 9, "accuracy": 0.5, "wilson95": [ 0.29031021505169974, 0.7096897849483003 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6235817419920153, "nll": 1.1743384371243408, "ece10": 0.1782765439935432, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.42554621848739493, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 2, "wrong_item": 1, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 4, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 4, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 3, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 5, "accuracy": 0.2777777777777778, "wilson95": [ 0.12499753379553993, 0.5087265656029746 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.2777777777777778, "brier": 1.2254289375366634, "nll": 3.2857388611575886, "ece10": 0.5889778615680563, "calibration_n": 18, "confident_wrong": 8, "macro_f1": 0.18666666666666668, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 1, "1": 2, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 4, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 2, "2": 0, "3": 1, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.7039945912499999, "nll": 1.0692159586709322, "ece10": 0.35883125000000005, "calibration_n": 16, "confident_wrong": 3, "macro_f1": 0.5465587044534412, "confusion_matrix": { "false": { "false": 3, "true": 7, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 5, "accuracy": 0.3333333333333333, "wilson95": [ 0.1517632443971473, 0.5828645226148023 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 5, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3333333333333333, "brier": 0.7398786733333331, "nll": 0.9718302983165885, "ece10": 0.3894066666666667, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.25, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 7, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.45826473734343875, "nll": 0.9400248876563944, "ece10": 0.2748075029952516, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5066666666666666, "confusion_matrix": { "billing": { "billing": 4, "tech": 1, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 1, "tech": 1, "sales": 1, "account": 0, "other": 1, "__failed__": 0 }, "account": { "billing": 1, "tech": 0, "sales": 0, "account": 2, "other": 0, "__failed__": 0 }, "other": { "billing": 1, "tech": 1, "sales": 0, "account": 0, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.4957225588235294, "nll": 0.6778770328437043, "ece10": 0.27424117647058827, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5296442687747036, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 7, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 8, "accuracy": 0.42105263157894735, "wilson95": [ 0.23141891282454238, 0.6372409646574496 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42105263157894735, "brier": 0.7207617926917743, "nll": 1.326167534432424, "ece10": 0.19028262754645786, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.275, "confusion_matrix": { "0": { "0": 3, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 4, "2": 1, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.5861162894117647, "nll": 0.8377120878769815, "ece10": 0.2940235294117647, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.37037037037037035, "confusion_matrix": { "false": { "false": 10, "true": 1, "__failed__": 0 }, "true": { "false": 6, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.4492951144444445, "nll": 0.6829588796457666, "ece10": 0.20951666666666668, "calibration_n": 18, "confident_wrong": 2, "macro_f1": 0.7142857142857142, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 9, "accuracy": 0.45, "wilson95": [ 0.2581978582556243, 0.6579146575496577 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.6567512897224781, "nll": 1.1735507974183308, "ece10": 0.1710970491180705, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.375, "confusion_matrix": { "0": { "0": 5, "1": 1, "2": 0, "3": 1, "__failed__": 0 }, "1": { "0": 2, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 2, "2": 1, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 15, "accuracy": 0.75, "wilson95": [ 0.531299122381256, 0.8881382985923343 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.339068696525039, "nll": 0.6979190109264758, "ece10": 0.23253900551684006, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7587301587301587, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 2, "reminder": 0, "music": 0, "small_talk": 3, "__failed__": 0 }, "reminder": { "device_control": 1, "information_request": 0, "reminder": 2, "music": 0, "small_talk": 1, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 10, "accuracy": 0.5, "wilson95": [ 0.2992980081982123, 0.7007019918017877 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.584338541, "nll": 0.7928659076010934, "ece10": 0.241585, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.40476190476190477, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 10, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 12, "accuracy": 0.5454545454545454, "wilson95": [ 0.3465979532073494, 0.7307970581918668 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5454545454545454, "brier": 0.5338075178127046, "nll": 0.9783956275822838, "ece10": 0.13700313782225865, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.5030303030303029, "confusion_matrix": { "0": { "0": 1, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 6, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 4, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 2, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 439, "accuracy": 0.48777777777777775, "wilson95": [ 0.4552428480919449, 0.5204165999392342 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 568, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 168, "head_accuracy": 0.5293569431500466, "cluster_bootstrap95": [ 0.45572571368025916, 0.5174247222285053 ], "independent_source_groups": 596, "brier": 0.645139117722733, "nll": 1.3243973856005011, "ece10": 0.2164619130500514, "calibration_n": 1073, "confident_wrong": 77 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 222, "accuracy": 0.3867595818815331, "wilson95": [ 0.34779711114770673, 0.42722768498700714 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 513, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 275, "head_accuracy": 0.5480769230769231, "brier": 0.6096417517059053, "nll": 1.2921150280932847, "ece10": 0.12618037899116655, "calibration_n": 936, "confident_wrong": 51 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 113, "accuracy": 0.16142857142857142, "wilson95": [ 0.13603236046436545, 0.1905205243369929 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 310, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 100, "head_accuracy": 0.2849264705882353, "cluster_bootstrap95": [ 0.13494119481218209, 0.18927222928103946 ], "independent_source_groups": 650, "brier": 0.8560350926151363, "nll": 1.6560835981325088, "ece10": 0.2681582138480542, "calibration_n": 1088, "confident_wrong": 68 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 7, "accuracy": 0.0175, "wilson95": [ 0.008502291107038103, 0.03567707292555804 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 719, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 7, "head_accuracy": 0.3595, "brier": 0.7502548091389216, "nll": 1.335514826707956, "ece10": 0.17699637561920184, "calibration_n": 2000, "confident_wrong": 13 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 25, "accuracy": 0.8928571428571429, "wilson95": [ 0.7280414845132785, 0.9628816331707 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 25, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8928571428571429, "brier": 0.24860420359473576, "nll": 0.4780478451444389, "ece10": 0.16481864833532933, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.8911858974358975, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 1, "inspection": 1, "purchase": 0, "work": 5, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 24, "accuracy": 0.8571428571428571, "wilson95": [ 0.685101978364747, 0.9430099468025059 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 24, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.2291500241132009, "nll": 0.4650473237894313, "ece10": 0.14860458850797442, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.855952380952381, "confusion_matrix": { "work": { "work": 5, "purchase": 0, "inspection": 1, "design": 1, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 6, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 1, "inspection": 0, "design": 6, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 21, "accuracy": 0.75, "wilson95": [ 0.5664432968100104, 0.8732350508070668 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3995814818747764, "nll": 0.775450487186126, "ece10": 0.23377092721367354, "calibration_n": 28, "confident_wrong": 1, "macro_f1": 0.7545787545787545, "confusion_matrix": { "design": { "design": 4, "inspection": 1, "purchase": 0, "work": 2, "__failed__": 0 }, "inspection": { "design": 1, "inspection": 6, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 6, "work": 1, "__failed__": 0 }, "work": { "design": 2, "inspection": 0, "purchase": 0, "work": 5, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 19, "accuracy": 0.6785714285714286, "wilson95": [ 0.49338843180857617, 0.8206675307750504 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6785714285714286, "brier": 0.44833144152548815, "nll": 0.8606307251139214, "ece10": 0.18559051692472658, "calibration_n": 28, "confident_wrong": 1, "macro_f1": 0.6865079365079365, "confusion_matrix": { "work": { "work": 3, "purchase": 0, "inspection": 0, "design": 4, "__failed__": 0 }, "purchase": { "work": 1, "purchase": 5, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 6, "design": 1, "__failed__": 0 }, "design": { "work": 1, "purchase": 0, "inspection": 1, "design": 5, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 33, "accuracy": 0.825, "wilson95": [ 0.6805000966821229, 0.9125458625396408 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 33, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.825, "brier": 0.41732805445454424, "nll": 0.8300222647437542, "ece10": 0.3015460197487602, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8206162464985995, "confusion_matrix": { "review": { "review": 4, "it": 0, "documents": 1, "procurement": 0, "maintenance": 3, "__failed__": 0 }, "it": { "review": 0, "it": 7, "documents": 0, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 6, "procurement": 1, "maintenance": 1, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 29, "accuracy": 0.725, "wilson95": [ 0.5716504419378943, 0.8389198375233267 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 29, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.725, "brier": 0.4554634846860378, "nll": 0.8955195431523013, "ece10": 0.2502010099803351, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.7249622926093514, "confusion_matrix": { "maintenance": { "maintenance": 7, "procurement": 0, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "procurement": { "maintenance": 1, "procurement": 7, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 1, "documents": 6, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 1, "documents": 1, "it": 5, "review": 0, "__failed__": 0 }, "review": { "maintenance": 3, "procurement": 0, "documents": 1, "it": 0, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 22, "accuracy": 0.55, "wilson95": [ 0.39829091798932054, 0.6929469218909508 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.6583780748695015, "nll": 1.312793856384692, "ece10": 0.20973659311494092, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.5453599540556062, "confusion_matrix": { "review": { "review": 4, "it": 0, "documents": 0, "procurement": 0, "maintenance": 4, "__failed__": 0 }, "it": { "review": 3, "it": 3, "documents": 0, "procurement": 1, "maintenance": 1, "__failed__": 0 }, "documents": { "review": 1, "it": 0, "documents": 3, "procurement": 1, "maintenance": 3, "__failed__": 0 }, "procurement": { "review": 2, "it": 1, "documents": 0, "procurement": 5, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 1, "documents": 0, "procurement": 0, "maintenance": 7, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 20, "accuracy": 0.5, "wilson95": [ 0.35199526933465386, 0.6480047306653461 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.67215346567304, "nll": 1.3466184792972056, "ece10": 0.155588168221247, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.45448397013614406, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 1, "procurement": 4, "documents": 0, "it": 0, "review": 3, "__failed__": 0 }, "documents": { "maintenance": 2, "procurement": 1, "documents": 2, "it": 0, "review": 3, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 3, "documents": 0, "it": 1, "review": 3, "__failed__": 0 }, "review": { "maintenance": 3, "procurement": 0, "documents": 0, "it": 0, "review": 5, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 239, "accuracy": 0.5975, "wilson95": [ 0.5487342037489487, 0.6444108988558637 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 239, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5975, "brier": 0.6954091995356698, "nll": 2.316165264713013, "ece10": 0.30329538490667896, "calibration_n": 400, "confident_wrong": 92, "macro_f1": 0.47767899336751946, "confusion_matrix": { "sadness": { "sadness": 87, "joy": 14, "love": 3, "anger": 8, "fear": 6, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 15, "joy": 87, "love": 12, "anger": 3, "fear": 1, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 7, "joy": 12, "love": 14, "anger": 3, "fear": 1, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 19, "joy": 5, "love": 3, "anger": 31, "fear": 4, "surprise": 2, "__failed__": 0 }, "fear": { "sadness": 21, "joy": 2, "love": 1, "anger": 7, "fear": 19, "surprise": 2, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 4, "love": 1, "anger": 2, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 234, "accuracy": 0.78, "wilson95": [ 0.7297473820840871, 0.8231725540301672 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 234, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 206, "head_accuracy": 0.78, "brier": 0.3748067941764854, "nll": 2.863946046813975, "ece10": 0.17954432501520273, "calibration_n": 300, "confident_wrong": 36 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 285, "accuracy": 0.7125, "wilson95": [ 0.6662901779980862, 0.7546670969098378 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 285, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 1, "head_accuracy": 0.7125, "brier": 0.5358312502499999, "nll": 3.2011909552288342, "ece10": 0.25725274999999953, "calibration_n": 400, "confident_wrong": 99, "macro_f1": 0.6885136004442067, "confusion_matrix": { "false": { "false": 198, "true": 111, "__failed__": 0 }, "true": { "false": 4, "true": 87, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 381, "accuracy": 0.9525, "wilson95": [ 0.9270084505901734, 0.9693829230372881 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 381, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9525, "brier": 0.0806960501240338, "nll": 0.16360697308573496, "ece10": 0.030394600698760994, "calibration_n": 400, "confident_wrong": 3, "macro_f1": 0.9468211676361148, "confusion_matrix": { "world": { "world": 99, "sports": 3, "business": 1, "sci_tech": 0, "__failed__": 0 }, "sports": { "world": 2, "sports": 121, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 0, "sports": 0, "business": 62, "sci_tech": 10, "__failed__": 0 }, "sci_tech": { "world": 0, "sports": 0, "business": 3, "sci_tech": 99, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 391, "accuracy": 0.9775, "wilson95": [ 0.9577973424412419, 0.988118416469505 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 391, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 101, "head_accuracy": 0.9775, "brier": 0.02944065374999997, "nll": 0.04817107510739655, "ece10": 0.01186875, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.9762719237537817, "confusion_matrix": { "true": { "true": 150, "false": 3, "__failed__": 0 }, "false": { "true": 6, "false": 241, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 179, "accuracy": 0.5966666666666667, "wilson95": [ 0.540271195643406, 0.6506178299198485 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 179, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5966666666666667, "brier": 0.7482596976170736, "nll": 7.815611704740443, "ece10": 0.3627327823207742, "calibration_n": 300, "confident_wrong": 96 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 399, "accuracy": 0.9975, "wilson95": [ 0.9859767149241766, 0.9995585522131983 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 399, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 91, "head_accuracy": 0.9975, "brier": 0.005125165350000001, "nll": 0.012453730545058099, "ece10": 0.008465249999999999, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.9974901960784314, "confusion_matrix": { "false": { "false": 187, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 212, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 212, "accuracy": 0.53, "wilson95": [ 0.4810361559058216, 0.5783931064341206 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 212, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 1, "head_accuracy": 0.53, "brier": 0.6016318265000004, "nll": 0.8867483849689616, "ece10": 0.2944955, "calibration_n": 400, "confident_wrong": 27, "macro_f1": 0.4003381072374087, "confusion_matrix": { "false": { "false": 199, "true": 1, "__failed__": 0 }, "true": { "false": 187, "true": 13, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 53, "accuracy": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 53, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.53, "brier": 0.5958024615999997, "nll": 0.820036667988229, "ece10": 0.16467600000000002, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.5160127690248172, "confusion_matrix": { "yes": { "yes": 18, "no": 36, "__failed__": 0 }, "no": { "yes": 11, "no": 35, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 50, "accuracy": 0.5, "wilson95": [ 0.4038315303659956, 0.5961684696340044 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 50, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.9482567181296087, "nll": 11.124913672940261, "ece10": 0.4537351656051146, "calibration_n": 100, "confident_wrong": 41, "macro_f1": 0.4705038017538017, "confusion_matrix": { "balance": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 1, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 4, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 7, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 1, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 2, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 1, "direct_deposit": 0, "close_account": 1, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "statement": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 2, "statement": 2, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 1, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 2, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 1, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 3, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 2, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 9, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 1, "branch_appointment": 1, "fee_refund": 1, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 5, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 2, "loan_payment": 2, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 140, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.4062862129750204, "nll": 0.7607971280413632, "ece10": 0.12391083801323302, "calibration_n": 200, "confident_wrong": 9 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 24, "accuracy": 0.24, "wilson95": [ 0.16691325556948772, 0.33232336349814473 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 99, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.495, "brier": 0.7879641559396634, "nll": 3.5319371936867934, "ece10": 0.31639558088299724, "calibration_n": 200, "confident_wrong": 36 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 70, "accuracy": 0.7, "wilson95": [ 0.6041514536665332, 0.7810511470506724 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 70, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.5654753520042951, "nll": 5.104187748574342, "ece10": 0.2731035313406568, "calibration_n": 100, "confident_wrong": 25, "macro_f1": 0.6561500196751001, "confusion_matrix": { "invoice": { "invoice": 6, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 2, "receipt": 12, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 3, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 0, "receipt": 1, "contract": 1, "resume": 0, "bank_statement": 10, "clinical_note": 2, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 6, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 2, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 2, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 1, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 2, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 4, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 1, "accuracy": 0.01, "wilson95": [ 0.0017674320641406505, 0.054486196178705315 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 194, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.485, "brier": 0.7468987777979195, "nll": 1.474015261709793, "ece10": 0.2565355186595327, "calibration_n": 400, "confident_wrong": 61 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 57, "accuracy": 0.57, "wilson95": [ 0.47215389549212305, 0.6626670147588988 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 57, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.57, "brier": 0.6688910873229011, "nll": 1.7607391972166082, "ece10": 0.24420999362248588, "calibration_n": 100, "confident_wrong": 12, "macro_f1": 0.5453013579700886, "confusion_matrix": { "politics": { "politics": 3, "business": 1, "technology": 3, "science": 3, "health": 1, "sports": 0, "entertainment": 0, "world": 1, "culture": 1, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 3, "technology": 7, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 1, "technology": 0, "science": 7, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 2, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 1, "health": 9, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 1, "health": 0, "sports": 7, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 0, "business": 0, "technology": 2, "science": 1, "health": 0, "sports": 0, "entertainment": 4, "world": 0, "culture": 2, "weather": 1, "__failed__": 0 }, "world": { "politics": 0, "business": 4, "technology": 1, "science": 1, "health": 0, "sports": 0, "entertainment": 1, "world": 0, "culture": 1, "weather": 1, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 2, "science": 1, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 5, "weather": 1, "__failed__": 0 }, "weather": { "politics": 0, "business": 2, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 6, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 28, "accuracy": 0.28, "wilson95": [ 0.20139685211186212, 0.3748802870992116 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.28, "brier": 0.9498805944238174, "nll": 2.6656065035587293, "ece10": 0.30873379433740583, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.2241779283518414, "confusion_matrix": { "biology": { "biology": 0, "medicine": 1, "chemistry": 1, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 2, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 0, "medicine": 6, "chemistry": 1, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 3, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 4, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 0, "medicine": 4, "chemistry": 4, "physics": 1, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 4, "economics": 1, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 2, "chemistry": 2, "physics": 0, "computer_science": 4, "mathematics": 0, "earth_science": 0, "engineering": 6, "economics": 1, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 2, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 1, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 4, "engineering": 6, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 7, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 3, "engineering": 2, "economics": 2, "other": 0, "__failed__": 0 }, "other": { "biology": 0, "medicine": 0, "chemistry": 2, "physics": 1, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 7, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 6, "accuracy": 0.06, "wilson95": [ 0.027786123963188192, 0.1247681544589592 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 60, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3, "brier": 0.7616429794257458, "nll": 1.429925927919855, "ece10": 0.3086621893930819, "calibration_n": 100, "confident_wrong": 13 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 4, "accuracy": 0.04, "wilson95": [ 0.01566330399154762, 0.0983707143588792 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 85, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.425, "brier": 0.30902623816532154, "nll": 0.5548159916982055, "ece10": 0.08276655034993552, "calibration_n": 100, "confident_wrong": 3 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 79, "accuracy": 0.79, "wilson95": [ 0.7002003116591013, 0.858343459380847 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 79, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.3116842897407037, "nll": 0.5438023536337231, "ece10": 0.05469927830701277, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.7858488027979553, "confusion_matrix": { "negative": { "negative": 34, "neutral": 2, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 12, "neutral": 19, "positive": 2, "__failed__": 0 }, "positive": { "negative": 2, "neutral": 3, "positive": 26, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 85, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.425, "brier": 0.34517014039999994, "nll": 0.534016196910454, "ece10": 0.118696, "calibration_n": 100, "confident_wrong": 7 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 26, "accuracy": 0.26, "wilson95": [ 0.18404698464748132, 0.3537098944918718 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 195, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.4849279728447662, "nll": 1.1873181372688792, "ece10": 0.1631527397251889, "calibration_n": 300, "confident_wrong": 13 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 41, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 100, "head_accuracy": 0.41, "brier": 1.0640934399324693, "nll": 12.557677148438609, "ece10": 0.5110331525725018, "calibration_n": 100, "confident_wrong": 36, "macro_f1": 0.34188793967814696, "confusion_matrix": { "order_status": { "order_status": 3, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 1, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 2, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 1, "refund_request": 1, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 2, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 2, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 3, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 1, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 3, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 4, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 3, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 2, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 4, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "other": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 2, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 31, "accuracy": 0.31, "wilson95": [ 0.22779697212376038, 0.40626055719489407 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 31, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.31, "brier": 1.1933132269512412, "nll": 10.620383344178437, "ece10": 0.5382049654460663, "calibration_n": 100, "confident_wrong": 40, "macro_f1": 0.30995977062153535, "confusion_matrix": { "billing": { "billing": 5, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 3, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 1, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 1, "shipping": 1, "returns": 0, "technical_outage": 1, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 2, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 3, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 3, "security": 0, "compliance": 2, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 1, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 3, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 2, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "pricing": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 3, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 4, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 1, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "other": { "billing": 5, "account_access": 0, "shipping": 1, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 4, "accuracy": 0.04, "wilson95": [ 0.01566330399154762, 0.0983707143588792 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 111, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.37, "brier": 0.902186102004676, "nll": 4.744571361504294, "ece10": 0.3616088774149999, "calibration_n": 300, "confident_wrong": 53 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 36, "accuracy": 0.36, "wilson95": [ 0.2727122040140433, 0.4576459754839126 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 36, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.36, "brier": 1.2134793853602057, "nll": 14.160017191448766, "ece10": 0.5819364334616243, "calibration_n": 100, "confident_wrong": 53, "macro_f1": 0.36061136327440674, "confusion_matrix": { "new_booking": { "new_booking": 5, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 7, "change_flight": 2, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 2, "change_flight": 0, "cancel_trip": 1, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 2, "change_flight": 0, "cancel_trip": 0, "seat": 3, "baggage": 0, "refund": 0, "hotel": 0, "car": 2, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 4, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 7, "refund": 0, "hotel": 1, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 5, "hotel": 0, "car": 2, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 4, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 6, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "car": { "new_booking": 2, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 2, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 4, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 2, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 2, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 0, "loyalty": 3, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 2, "change_flight": 2, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 2, "disruption": 0, "other": 0, "__failed__": 0 }, "other": { "new_booking": 7, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 1, "visa_docs": 0, "loyalty": 2, "disruption": 1, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 96, "accuracy": 0.96, "wilson95": [ 0.9016292856411208, 0.9843366960084523 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 96, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.96, "brier": 0.06975929864743771, "nll": 0.16915019825919397, "ece10": 0.023695184235831834, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.9595472804506419, "confusion_matrix": { "business": { "business": 24, "science and technology": 1, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 0, "science and technology": 22, "sports": 1, "world": 2, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 45, "accuracy": 0.45, "wilson95": [ 0.3561453797951198, 0.5475539700255787 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 45, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 100, "head_accuracy": 0.45, "brier": 0.9113759346210644, "nll": 7.858033268038419, "ece10": 0.40773806664392515, "calibration_n": 100, "confident_wrong": 30, "macro_f1": 0.3775352733686068, "confusion_matrix": { "refund not showing up": { "refund not showing up": 1, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 1, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 1, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 1, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 2, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 1, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 1, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 2, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 1, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 1, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 2, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 1, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 2, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 1, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 1, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 1, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 1, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 44, "accuracy": 0.44, "wilson95": [ 0.34672026630160796, 0.5377189534832302 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 44, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.44, "brier": 1.0028066980397166, "nll": 3.2092905053473157, "ece10": 0.45168956035124364, "calibration_n": 100, "confident_wrong": 40, "macro_f1": 0.42518047769539674, "confusion_matrix": { "anger": { "anger": 10, "fear": 0, "joy": 0, "love": 0, "sadness": 7, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 1, "fear": 8, "joy": 1, "love": 0, "sadness": 7, "surprise": 0, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 13, "love": 1, "sadness": 3, "surprise": 0, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 8, "love": 5, "sadness": 4, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 4, "fear": 2, "joy": 3, "love": 0, "sadness": 7, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 0, "joy": 8, "love": 0, "sadness": 6, "surprise": 1, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1962, "accuracy": 0.6655359565807327, "wilson95": [ 0.6482990705972358, 0.6823419931643413 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1962, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6655359565807327, "brier": 0.49848601060576436, "nll": 1.624779521497024, "ece10": 0.1809816346012473, "calibration_n": 2948, "confident_wrong": 229, "macro_f1": 0.6904815123681389, "confusion_matrix": { "alarm": { "alarm": 94, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 3, "audio": 24, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 25, "iot": 4, "lists": 0, "music": 4, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 20, "audio": 0, "calendar": 289, "cooking": 2, "datetime": 18, "email": 2, "general": 40, "iot": 5, "lists": 2, "music": 2, "news": 5, "play": 0, "qa": 2, "recommendation": 5, "social": 2, "takeaway": 5, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 65, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 90, "email": 0, "general": 7, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 2, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 0, "email": 210, "general": 46, "iot": 3, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "general": { "alarm": 1, "audio": 1, "calendar": 9, "cooking": 4, "datetime": 8, "email": 0, "general": 115, "iot": 7, "lists": 1, "music": 1, "news": 5, "play": 3, "qa": 3, "recommendation": 21, "social": 0, "takeaway": 1, "transport": 2, "weather": 5, "__failed__": 0 }, "iot": { "alarm": 11, "audio": 4, "calendar": 0, "cooking": 6, "datetime": 0, "email": 0, "general": 34, "iot": 156, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 8, "cooking": 2, "datetime": 0, "email": 0, "general": 14, "iot": 3, "lists": 105, "music": 4, "news": 0, "play": 1, "qa": 0, "recommendation": 4, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 14, "iot": 0, "lists": 1, "music": 58, "news": 1, "play": 4, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 18, "iot": 0, "lists": 0, "music": 0, "news": 100, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 3, "weather": 1, "__failed__": 0 }, "play": { "alarm": 3, "audio": 8, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 59, "iot": 4, "lists": 1, "music": 68, "news": 1, "play": 235, "qa": 1, "recommendation": 5, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 4, "email": 0, "general": 187, "iot": 8, "lists": 0, "music": 2, "news": 18, "play": 3, "qa": 48, "recommendation": 10, "social": 0, "takeaway": 0, "transport": 4, "weather": 1, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 8, "cooking": 0, "datetime": 0, "email": 0, "general": 11, "iot": 0, "lists": 0, "music": 0, "news": 11, "play": 2, "qa": 0, "recommendation": 59, "social": 0, "takeaway": 1, "transport": 2, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 22, "iot": 1, "lists": 1, "music": 0, "news": 3, "play": 0, "qa": 0, "recommendation": 1, "social": 75, "takeaway": 1, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 2, "datetime": 3, "email": 0, "general": 10, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 38, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 1, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 2, "email": 1, "general": 15, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 8, "social": 0, "takeaway": 0, "transport": 90, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 3, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 23, "iot": 6, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 1, "weather": 111, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 621, "accuracy": 0.21065128900949798, "wilson95": [ 0.19631283861441726, 0.22574284296626865 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 621, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.21065128900949798, "brier": 1.1120353437172774, "nll": 3.3946049787823496, "ece10": 0.4694431218170974, "calibration_n": 2948, "confident_wrong": 167, "macro_f1": 0.2558574073858679, "confusion_matrix": { "alarm": { "alarm": 77, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 14, "iot": 1, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 2, "calendar": 0, "cooking": 1, "datetime": 2, "email": 0, "general": 48, "iot": 2, "lists": 1, "music": 3, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 16, "audio": 0, "calendar": 34, "cooking": 4, "datetime": 7, "email": 2, "general": 294, "iot": 6, "lists": 19, "music": 1, "news": 0, "play": 1, "qa": 0, "recommendation": 15, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 14, "datetime": 0, "email": 1, "general": 45, "iot": 1, "lists": 4, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 3, "transport": 0, "weather": 1, "__failed__": 0 }, "datetime": { "alarm": 3, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 15, "email": 1, "general": 70, "iot": 0, "lists": 0, "music": 2, "news": 6, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 3, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 2, "email": 65, "general": 184, "iot": 1, "lists": 1, "music": 0, "news": 3, "play": 3, "qa": 0, "recommendation": 2, "social": 2, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "general": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 3, "datetime": 4, "email": 3, "general": 142, "iot": 7, "lists": 6, "music": 1, "news": 7, "play": 3, "qa": 0, "recommendation": 8, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "iot": { "alarm": 3, "audio": 0, "calendar": 1, "cooking": 2, "datetime": 1, "email": 0, "general": 159, "iot": 22, "lists": 10, "music": 4, "news": 0, "play": 4, "qa": 0, "recommendation": 5, "social": 0, "takeaway": 3, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 76, "iot": 0, "lists": 61, "music": 1, "news": 0, "play": 1, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 49, "iot": 1, "lists": 0, "music": 29, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 2, "email": 2, "general": 101, "iot": 0, "lists": 0, "music": 0, "news": 13, "play": 1, "qa": 0, "recommendation": 3, "social": 0, "takeaway": 1, "transport": 1, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 0, "email": 2, "general": 260, "iot": 2, "lists": 4, "music": 45, "news": 1, "play": 50, "qa": 0, "recommendation": 17, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 255, "iot": 3, "lists": 1, "music": 1, "news": 8, "play": 2, "qa": 1, "recommendation": 9, "social": 0, "takeaway": 2, "transport": 0, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 2, "datetime": 0, "email": 0, "general": 65, "iot": 3, "lists": 1, "music": 3, "news": 1, "play": 7, "qa": 0, "recommendation": 10, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "social": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 43, "iot": 1, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 57, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 4, "datetime": 2, "email": 0, "general": 29, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 3, "social": 0, "takeaway": 15, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 2, "datetime": 1, "email": 2, "general": 85, "iot": 4, "lists": 2, "music": 0, "news": 0, "play": 2, "qa": 0, "recommendation": 14, "social": 0, "takeaway": 2, "transport": 8, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 3, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 13, "email": 0, "general": 113, "iot": 3, "lists": 1, "music": 0, "news": 3, "play": 1, "qa": 1, "recommendation": 4, "social": 0, "takeaway": 2, "transport": 0, "weather": 6, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 626, "accuracy": 0.5336743393009378, "wilson95": [ 0.5050624307833703, 0.5620664075218041 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 626, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5336743393009378, "cluster_bootstrap95": [ 0.5054347826086957, 0.5628371964199572 ], "independent_source_groups": 47, "brier": 0.6216222629740593, "nll": 1.0526833731605927, "ece10": 0.14813270665069117, "calibration_n": 1173, "confident_wrong": 16, "macro_f1": 0.46971504794890356, "confusion_matrix": { "negative": { "negative": 52, "neutral": 100, "positive": 30, "__failed__": 0 }, "neutral": { "negative": 66, "neutral": 397, "positive": 127, "__failed__": 0 }, "positive": { "negative": 48, "neutral": 176, "positive": 177, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "nev-2b": { "label": "Nev 2B Q8", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 177, "accuracy": 0.885, "wilson95": [ 0.8333528310151243, 0.9221362676743804 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 177, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.885, "brier": 0.1760271172342542, "nll": 0.33444002137198814, "ece10": 0.05358872085165984, "calibration_n": 200, "confident_wrong": 6, "macro_f1": 0.8877158683610297, "confusion_matrix": { "world": { "world": 45, "sports": 0, "business": 1, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 3, "sports": 0, "business": 41, "sci_tech": 7, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 5, "sci_tech": 45, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 123, "accuracy": 0.615, "wilson95": [ 0.5459986722024608, 0.6796669027307678 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 123, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.615, "brier": 0.611011696887005, "nll": 1.2252133958333065, "ece10": 0.21983735852182493, "calibration_n": 200, "confident_wrong": 22, "macro_f1": 0.42964361312057076, "confusion_matrix": { "sadness": { "sadness": 43, "joy": 4, "love": 1, "anger": 6, "fear": 2, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 14, "joy": 55, "love": 6, "anger": 2, "fear": 2, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 2, "joy": 9, "love": 1, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 9, "joy": 4, "love": 1, "anger": 14, "fear": 0, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 5, "joy": 3, "love": 0, "anger": 1, "fear": 10, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 166, "accuracy": 0.83, "wilson95": [ 0.7718411637168737, 0.8757209208741302 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 166, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.83, "brier": 0.26816104767580917, "nll": 0.42609997722909837, "ece10": 0.061009847486237945, "calibration_n": 200, "confident_wrong": 5, "macro_f1": 0.8186666666666667, "confusion_matrix": { "false": { "false": 108, "true": 1, "__failed__": 0 }, "true": { "false": 33, "true": 58, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 136, "accuracy": 0.68, "wilson95": [ 0.6124810156588646, 0.7407346668453195 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 136, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.68, "brier": 0.44357692894937645, "nll": 0.6452879495030862, "ece10": 0.19291109764531708, "calibration_n": 200, "confident_wrong": 13, "macro_f1": 0.5482778091473743, "confusion_matrix": { "true": { "true": 14, "false": 63, "__failed__": 0 }, "false": { "true": 1, "false": 122, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.40090663340870236, "nll": 0.7531394670291366, "ece10": 0.09034591686688856, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.7733333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 6, "accuracy": 0.5, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.5326178017213462, "nll": 0.9790900205309203, "ece10": 0.3351836939337052, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.5333333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 1, "review": 1, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 2, "documents": 1, "it": 1, "review": 0, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.012685334756385168, "nll": 0.08393204887907106, "ece10": 0.0795675503945851, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03109087499494705, "nll": 0.12223674127804703, "ece10": 0.11142857251641904, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.1663000399310256, "nll": 0.30537952434188215, "ece10": 0.04927288971104308, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.8702328101051742, "confusion_matrix": { "label_000": { "label_000": 24, "label_001": 1, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 7, "label_001": 14, "label_002": 1, "label_003": 3, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.7818009747578982, "nll": 1.5889221916350404, "ece10": 0.3037813037573174, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.4983502512812858, "confusion_matrix": { "label_000": { "label_000": 7, "label_001": 3, "label_002": 0, "label_003": 0, "label_004": 7, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 8, "label_002": 0, "label_003": 0, "label_004": 9, "label_005": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 12, "label_003": 1, "label_004": 4, "label_005": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 4, "label_003": 5, "label_004": 8, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 1, "label_002": 4, "label_003": 1, "label_004": 10, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 0, "label_002": 5, "label_003": 0, "label_004": 4, "label_005": 6, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "label_000": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_001": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0007872943656512928, "nll": 0.012888572815466235, "ece10": 0.012606766702710994, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 75, "accuracy": 0.6756756756756757, "wilson95": [ 0.5839813435379284, 0.7556172683265554 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 75, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6756756756756757, "brier": 0.4213799938209524, "nll": 0.7614553969925631, "ece10": 0.0701861758742815, "calibration_n": 111, "confident_wrong": 2 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 63, "accuracy": 0.875, "wilson95": [ 0.7791894627583916, 0.9328221577942649 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 63, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "cluster_bootstrap95": [ 0.8055555555555556, 0.9444444444444444 ], "independent_source_groups": 36, "brier": 0.15597637562975572, "nll": 0.2795286978261471, "ece10": 0.10259353571422475, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.12173823097398734, "nll": 0.23988243300556814, "ece10": 0.06226514534636898, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9467787114845938, "confusion_matrix": { "false": { "false": 10, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.1627648782775599, "nll": 0.2770558001954757, "ece10": 0.14250302762760497, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.26625757836678815, "nll": 0.4183426242901844, "ece10": 0.20647970362593515, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8633540372670807, "confusion_matrix": { "false": { "false": 9, "true": 1, "__failed__": 0 }, "true": { "false": 2, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.21147710130979405, "nll": 0.34306934468307404, "ece10": 0.13671556602135254, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.886904761904762, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 3, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 1, "feature_request": 0, "praise": 0, "pricing_billing": 3, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.32145927717158673, "nll": 0.6071124874216193, "ece10": 0.1612775401919418, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7880952380952381, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 3, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.013491016302307231, "nll": 0.06858413819444112, "ece10": 0.06369500162961472, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.2295750131145623, "nll": 0.45854996782377394, "ece10": 0.25867057427529216, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9523809523809524, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 1, "performance": 0, "security": 3, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 4, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.23264428863672487, "nll": 0.4236735498468376, "ece10": 0.1866196245734019, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.837037037037037, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 0, "chore": 1, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.5013053842796905, "nll": 0.6753619555254995, "ece10": 0.31357186909771867, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.43333333333333335, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 8, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.2116347441979174, "nll": 0.3562847409157747, "ece10": 0.12192328666496344, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.31008841788268166, "nll": 0.6272813172603516, "ece10": 0.21549097011842577, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.4928571428571429, "confusion_matrix": { "harassment": { "harassment": 5, "threat": 0, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 1, "none": 1, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 3, "none": 1, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 5, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.21965716794865478, "nll": 0.4502404440605235, "ece10": 0.13616338843617556, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8338461538461539, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 3, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 1, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02816989675512794, "nll": 0.06655015259463547, "ece10": 0.056189827306697755, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.10626306722581842, "nll": 0.2050876717183229, "ece10": 0.10363607170556617, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "address_issue": { "address_issue": 3, "customs_hold": 1, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.49908242038158324, "nll": 0.730052301529952, "ece10": 0.2792346148287809, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.6640316205533596, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.008947102901467682, "nll": 0.058320637413766696, "ece10": 0.05545233134647834, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.14566696790991174, "nll": 0.4007851527751802, "ece10": 0.10419029663539173, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.11440857459607025, "nll": 0.23293599980686816, "ece10": 0.16804962019417488, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 1, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 2, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.2919362399180387, "nll": 0.46385445936960107, "ece10": 0.14171545552760684, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.8285714285714285, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 7, "accuracy": 0.4375, "wilson95": [ 0.23098652405492354, 0.6682144360118811 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4375, "brier": 0.7375464158379701, "nll": 1.328234414108267, "ece10": 0.3021799307774954, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.41507936507936505, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.24561311676746594, "nll": 0.44271950385505326, "ece10": 0.18093314788229553, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8222222222222223, "confusion_matrix": { "0": { "0": 3, "1": 2, "2": 0, "__failed__": 0 }, "1": { "0": 1, "1": 6, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 16, "accuracy": 0.8, "wilson95": [ 0.5839825677481064, 0.919342337420202 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.28350382376246797, "nll": 0.587073417661931, "ece10": 0.13374461833300777, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7744810744810745, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 1, "harassment": 0, "boosting": 0, "smurfing": 1, "other": 1, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 7, "accuracy": 0.3333333333333333, "wilson95": [ 0.17194752605083477, 0.5462654802574476 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3333333333333333, "brier": 0.9702435582800852, "nll": 1.908802106721872, "ece10": 0.4236750695379089, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.24333333333333335, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 0, "punctuation": 0, "none": 1, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04625053551803949, "nll": 0.15716321996011945, "ece10": 0.14400249204336413, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.022041854182074, "nll": 0.11553256274741609, "ece10": 0.10663115023795816, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 11, "accuracy": 0.7857142857142857, "wilson95": [ 0.524107694133997, 0.9242861328730683 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.34833558193319863, "nll": 0.6710083979619591, "ece10": 0.17532257655414155, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.6992063492063492, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.4156202021475144, "nll": 0.7576071216540573, "ece10": 0.12683040391226444, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5714285714285714, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 1, "2": 1, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 1, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3787175500968578, "nll": 0.6990056207582158, "ece10": 0.2628958518598261, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7541666666666667, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 2, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.2991294976950742, "nll": 0.4743002771781883, "ece10": 0.22236545885501366, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8819444444444444, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.11615857010845324, "nll": 0.30257556015243153, "ece10": 0.04068727724650838, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.9377777777777778, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "sports": 2, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.06057128290409553, "nll": 0.18289676735762747, "ece10": 0.15715342433644247, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 1, "infra": 3, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.3326893433288365, "nll": 0.5053141604934651, "ece10": 0.20352534029193672, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7571428571428571, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.13948057560872215, "nll": 0.27507829801283273, "ece10": 0.16269642333647624, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9372549019607843, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3865924212245074, "nll": 0.6677720639527519, "ece10": 0.11380100314682903, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7316017316017316, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 3, "1": 2, "2": 0, "__failed__": 0 }, "2": { "0": 1, "1": 0, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.07712705323510416, "nll": 0.22301635367838765, "ece10": 0.09802009992658073, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9523809523809524, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 0, "mexican": 1, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.5739880739810692, "nll": 0.7781565296846381, "ece10": 0.21439980434730174, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5142857142857142, "confusion_matrix": { "false": { "false": 3, "true": 5, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.23219656051823753, "nll": 0.39480552213650627, "ece10": 0.1960255567071029, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.838095238095238, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 2, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 0, "changed_mind": 3, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 11, "accuracy": 0.6111111111111112, "wilson95": [ 0.3861904160223791, 0.7969475342783638 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6111111111111112, "brier": 0.4474560024884251, "nll": 1.0064514731665524, "ece10": 0.1404730793489064, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5603174603174603, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 2, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.4323890837135806, "nll": 0.6116524665704293, "ece10": 0.26291256597284385, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.75, "confusion_matrix": { "false": { "false": 6, "true": 4, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 7, "accuracy": 0.4666666666666667, "wilson95": [ 0.24809535365737387, 0.698830199745016 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 7, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4666666666666667, "brier": 0.5784459776664007, "nll": 0.7841970948413162, "ece10": 0.37100343959159193, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.4, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 6, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06972903807022386, "nll": 0.21622494813466786, "ece10": 0.18782230357639096, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 4, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.28474400625768426, "nll": 0.4539213760130753, "ece10": 0.04005001084669345, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8210526315789474, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 9, "accuracy": 0.47368421052631576, "wilson95": [ 0.2732980516117956, 0.6829219075488682 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47368421052631576, "brier": 0.6018334681524945, "nll": 0.9527003239162429, "ece10": 0.22879503266155246, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.38920454545454547, "confusion_matrix": { "0": { "0": 5, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 4, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 6, "accuracy": 0.35294117647058826, "wilson95": [ 0.17309720397243744, 0.5869963651478132 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35294117647058826, "brier": 0.6244090975711972, "nll": 0.8226725438992326, "ece10": 0.3823858621382745, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.315018315018315, "confusion_matrix": { "false": { "false": 1, "true": 10, "__failed__": 0 }, "true": { "false": 1, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.1651101977659388, "nll": 0.2710113194387876, "ece10": 0.13320578053487303, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8888888888888888, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 13, "accuracy": 0.65, "wilson95": [ 0.43285427668523624, 0.818808175898918 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.4058330251031227, "nll": 0.6874108537444066, "ece10": 0.1618222962628826, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.5944444444444444, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 1, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.03013726656262422, "nll": 0.07094404743651081, "ece10": 0.05750867641855042, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 1, "reminder": 0, "music": 0, "small_talk": 3, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 9, "accuracy": 0.45, "wilson95": [ 0.2581978582556243, 0.6579146575496577 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.7709994729887885, "nll": 1.0713923204666882, "ece10": 0.4149522493391075, "calibration_n": 20, "confident_wrong": 2, "macro_f1": 0.3103448275862069, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 11, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.2590756496202014, "nll": 0.5342348280566572, "ece10": 0.27845134715465103, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.912878787878788, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 461, "accuracy": 0.5122222222222222, "wilson95": [ 0.47958340006076583, 0.5447571519080551 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 605, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5638397017707363, "cluster_bootstrap95": [ 0.48375, 0.5419764740379843 ], "independent_source_groups": 596, "brier": 0.5828491509015197, "nll": 0.9962418233758735, "ece10": 0.19373884348454004, "calibration_n": 1073, "confident_wrong": 35 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 350, "accuracy": 0.6097560975609756, "wilson95": [ 0.5692466124675816, 0.6488062774326188 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 690, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7371794871794872, "brier": 0.38256151647577374, "nll": 0.723012463240113, "ece10": 0.08592997116515211, "calibration_n": 936, "confident_wrong": 37 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 219, "accuracy": 0.31285714285714283, "wilson95": [ 0.2796096359016959, 0.34814744388302116 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 491, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45128676470588236, "cluster_bootstrap95": [ 0.2803976001934353, 0.3468560071804077 ], "independent_source_groups": 650, "brier": 0.6834173126608196, "nll": 1.2779073885941352, "ece10": 0.16386363993048367, "calibration_n": 1088, "confident_wrong": 6 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 33, "accuracy": 0.0825, "wilson95": [ 0.05934587622971528, 0.1135968895394223 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1142, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.571, "brier": 0.5428038144339676, "nll": 0.9380937121383471, "ece10": 0.06969343251594555, "calibration_n": 2000, "confident_wrong": 15 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02721500129451599, "nll": 0.09350906019301353, "ece10": 0.08314115852074008, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.039701227955636645, "nll": 0.12199466118266646, "ece10": 0.1067725178185318, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.040825675759402644, "nll": 0.11932872561843451, "ece10": 0.1002080322604284, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 0, "purchase": 6, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.06784271743560837, "nll": 0.1700387492122524, "ece10": 0.09828068816240279, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 6, "inspection": 0, "design": 1, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.17311418329434208, "nll": 0.3536769990602261, "ece10": 0.21713241540356626, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.868235294117647, "confusion_matrix": { "review": { "review": 4, "it": 0, "documents": 0, "procurement": 1, "maintenance": 3, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 7, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 34, "accuracy": 0.85, "wilson95": [ 0.709276756335103, 0.9293881228267964 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.1650437783315995, "nll": 0.3414176783927922, "ece10": 0.15414611748095328, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8395321637426901, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 1, "documents": 6, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 1, "documents": 1, "it": 0, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 33, "accuracy": 0.825, "wilson95": [ 0.6805000966821229, 0.9125458625396408 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 33, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.825, "brier": 0.294392305344545, "nll": 0.5767247571924906, "ece10": 0.14620179564197516, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8055555555555556, "confusion_matrix": { "review": { "review": 3, "it": 1, "documents": 1, "procurement": 1, "maintenance": 2, "__failed__": 0 }, "it": { "review": 1, "it": 7, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 1, "procurement": 7, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 34, "accuracy": 0.85, "wilson95": [ 0.709276756335103, 0.9293881228267964 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.26377160148611783, "nll": 0.5237210155795747, "ece10": 0.12989333683238147, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8278817587641119, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 7, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 1, "documents": 1, "it": 1, "review": 3, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 237, "accuracy": 0.5925, "wilson95": [ 0.5436883047891221, 0.639551920759033 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 237, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5925, "brier": 0.5891122574437474, "nll": 1.2016526854312508, "ece10": 0.21268478048430384, "calibration_n": 400, "confident_wrong": 24, "macro_f1": 0.4838373107830655, "confusion_matrix": { "sadness": { "sadness": 88, "joy": 16, "love": 2, "anger": 9, "fear": 4, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 18, "joy": 86, "love": 8, "anger": 2, "fear": 1, "surprise": 3, "__failed__": 0 }, "love": { "sadness": 14, "joy": 15, "love": 8, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 24, "joy": 5, "love": 2, "anger": 28, "fear": 3, "surprise": 2, "__failed__": 0 }, "fear": { "sadness": 17, "joy": 2, "love": 1, "anger": 5, "fear": 25, "surprise": 2, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 3, "love": 1, "anger": 1, "fear": 1, "surprise": 2, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 268, "accuracy": 0.8933333333333333, "wilson95": [ 0.8532967116826325, 0.9234241509540578 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 268, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8933333333333333, "brier": 0.15557523375320806, "nll": 0.3695673547618735, "ece10": 0.036508731110414784, "calibration_n": 300, "confident_wrong": 5 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 346, "accuracy": 0.865, "wilson95": [ 0.8280190157519688, 0.8950370093839947 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 346, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.865, "brier": 0.189327904287927, "nll": 0.319680091527013, "ece10": 0.08724552253278972, "calibration_n": 400, "confident_wrong": 3, "macro_f1": 0.8223450454007106, "confusion_matrix": { "false": { "false": 271, "true": 38, "__failed__": 0 }, "true": { "false": 16, "true": 75, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 355, "accuracy": 0.8875, "wilson95": [ 0.8527764306768796, 0.9148515412140404 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 355, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8875, "brier": 0.18644923505402705, "nll": 0.36918686631682973, "ece10": 0.043066212120373024, "calibration_n": 400, "confident_wrong": 19, "macro_f1": 0.8786138336616935, "confusion_matrix": { "world": { "world": 87, "sports": 6, "business": 9, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 2, "sports": 121, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 4, "sports": 0, "business": 66, "sci_tech": 2, "__failed__": 0 }, "sci_tech": { "world": 5, "sports": 0, "business": 16, "sci_tech": 81, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 287, "accuracy": 0.7175, "wilson95": [ 0.6714724243160557, 0.7593897276485253 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 287, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7175, "brier": 0.403128763948268, "nll": 0.5903029752763391, "ece10": 0.1496156299341185, "calibration_n": 400, "confident_wrong": 21, "macro_f1": 0.6194997937554192, "confusion_matrix": { "true": { "true": 42, "false": 111, "__failed__": 0 }, "false": { "true": 2, "false": 245, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 249, "accuracy": 0.83, "wilson95": [ 0.7833857374317239, 0.8682699015600757 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 249, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.83, "brier": 0.2612806868290532, "nll": 0.6393764889031855, "ece10": 0.042440013999309856, "calibration_n": 300, "confident_wrong": 3 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 337, "accuracy": 0.8425, "wilson95": [ 0.8035652075632727, 0.8749188708177342 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 337, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8425, "brier": 0.24598140954306116, "nll": 0.39852303306492276, "ece10": 0.03173168026365511, "calibration_n": 400, "confident_wrong": 8, "macro_f1": 0.841548299473714, "confusion_matrix": { "false": { "false": 184, "true": 4, "__failed__": 0 }, "true": { "false": 59, "true": 153, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 267, "accuracy": 0.6675, "wilson95": [ 0.6199313395835853, 0.7118820418144254 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 267, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6675, "brier": 0.4651837227816085, "nll": 0.6671908462170216, "ece10": 0.17840519621304138, "calibration_n": 400, "confident_wrong": 19, "macro_f1": 0.6363909753880432, "confusion_matrix": { "false": { "false": 192, "true": 8, "__failed__": 0 }, "true": { "false": 125, "true": 75, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 75, "accuracy": 0.75, "wilson95": [ 0.656955364519384, 0.8245478863771232 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 75, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.32832233332731725, "nll": 0.5134706477982554, "ece10": 0.07885385087971007, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.7406369955389562, "confusion_matrix": { "yes": { "yes": 47, "no": 7, "__failed__": 0 }, "no": { "yes": 18, "no": 28, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 52, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.7023328745882667, "nll": 1.6690930673110964, "ece10": 0.2375430595192391, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.5226642326971275, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 3, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 6, "wire": 2, "statement": 0, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 4, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 4, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 6, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 4, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 1, "branch_appointment": 1, "fee_refund": 5, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 4, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 4, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 4, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 2, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 144, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.72, "brier": 0.3327739367736012, "nll": 0.5417493487888545, "ece10": 0.09379631803506296, "calibration_n": 200, "confident_wrong": 1 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 128, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.64, "brier": 0.5429301504403444, "nll": 1.0640514692136005, "ece10": 0.18436878074570653, "calibration_n": 200, "confident_wrong": 9 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.3352918741224691, "nll": 0.93003619181697, "ece10": 0.09633976922577844, "calibration_n": 100, "confident_wrong": 9, "macro_f1": 0.7649910110242928, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 2, "receipt": 13, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 13, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 1, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 7, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 2, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 9, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 5, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 4, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 7, "accuracy": 0.07, "wilson95": [ 0.03431926106727268, 0.137495147390735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 217, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5425, "brier": 0.6645722311192516, "nll": 1.3029631800595787, "ece10": 0.21784277256203605, "calibration_n": 400, "confident_wrong": 21 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 60, "accuracy": 0.6, "wilson95": [ 0.5020025867910618, 0.6905987135675411 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 60, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5978285003226739, "nll": 1.4677610934540244, "ece10": 0.1983588005348461, "calibration_n": 100, "confident_wrong": 9, "macro_f1": 0.581320241331683, "confusion_matrix": { "politics": { "politics": 2, "business": 0, "technology": 2, "science": 3, "health": 5, "sports": 0, "entertainment": 1, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 1, "technology": 8, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 2, "technology": 0, "science": 6, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "science": 2, "health": 0, "sports": 5, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 0, "business": 0, "technology": 1, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 2, "weather": 1, "__failed__": 0 }, "world": { "politics": 0, "business": 3, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 2, "culture": 0, "weather": 2, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 2, "world": 0, "culture": 6, "weather": 1, "__failed__": 0 }, "weather": { "politics": 0, "business": 2, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 7, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 43, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43, "brier": 0.7954205742462951, "nll": 1.9059732312403674, "ece10": 0.22688101053951573, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.4061285963183602, "confusion_matrix": { "biology": { "biology": 6, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 0, "medicine": 8, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 3, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 1, "medicine": 3, "chemistry": 1, "physics": 5, "computer_science": 0, "mathematics": 0, "earth_science": 5, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 3, "mathematics": 0, "earth_science": 3, "engineering": 7, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 2, "mathematics": 0, "earth_science": 2, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 8, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 5, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 0, "economics": 5, "other": 0, "__failed__": 0 }, "other": { "biology": 2, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 1, "mathematics": 0, "earth_science": 4, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 11, "accuracy": 0.11, "wilson95": [ 0.06254196357064568, 0.18631296503080313 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.345, "brier": 0.6348900839184016, "nll": 1.2179266520985985, "ece10": 0.24717591212485618, "calibration_n": 100, "confident_wrong": 7 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 13, "accuracy": 0.13, "wilson95": [ 0.07757167427240512, 0.20980351440076428 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 92, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.46, "brier": 0.34172369087291604, "nll": 0.5984626512564832, "ece10": 0.09863719684264667, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 86, "accuracy": 0.86, "wilson95": [ 0.7786281178903616, 0.9147365634006086 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.24265565989547988, "nll": 0.4591204578673204, "ece10": 0.15755969841585996, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8621569600249902, "confusion_matrix": { "negative": { "negative": 32, "neutral": 4, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 4, "neutral": 29, "positive": 0, "__failed__": 0 }, "positive": { "negative": 1, "neutral": 5, "positive": 25, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 21, "accuracy": 0.21, "wilson95": [ 0.141656540619153, 0.29979968834089865 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 106, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.53, "brier": 0.25471441785546395, "nll": 0.38518683749335075, "ece10": 0.08847901298033581, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 235, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7833333333333333, "brier": 0.3073068247760868, "nll": 0.6479741877809088, "ece10": 0.05846223786404366, "calibration_n": 300, "confident_wrong": 8 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "order_status": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "refund_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 6 }, "shipping_delay": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "pricing_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "speak_to_human": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 7 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 9 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 7 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "promo_code": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "partner_inquiry": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 42, "accuracy": 0.42, "wilson95": [ 0.3279838267435473, 0.5179351329695703 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 42, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42, "brier": 0.8741345685448849, "nll": 2.092235533713736, "ece10": 0.3431111288985483, "calibration_n": 100, "confident_wrong": 10, "macro_f1": 0.3916765199373895, "confusion_matrix": { "billing": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 6, "shipping": 1, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 5, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 2, "account_access": 1, "shipping": 2, "returns": 0, "technical_outage": 1, "onboarding": 0, "integrations": 4, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 2, "integrations": 4, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 7, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 2, "compliance": 1, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 1, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 1, "pricing": 1, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 5, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 4, "account_access": 0, "shipping": 3, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 12, "accuracy": 0.12, "wilson95": [ 0.06999406437019488, 0.19812099426711421 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 136, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4533333333333333, "brier": 0.692947503898705, "nll": 1.486367286355028, "ece10": 0.17708262163228358, "calibration_n": 300, "confident_wrong": 1 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 60, "accuracy": 0.6, "wilson95": [ 0.5020025867910618, 0.6905987135675411 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 60, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.5908813188329881, "nll": 1.4733622862738895, "ece10": 0.1980891291761273, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.5614390840784029, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 6, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 2, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 5, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 9, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 2, "loyalty": 2, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 6, "disruption": 1, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 1, "disruption": 5, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 1, "hotel": 3, "car": 2, "visa_docs": 1, "loyalty": 2, "disruption": 1, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 91, "accuracy": 0.91, "wilson95": [ 0.837737871472837, 0.9519274599974348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 91, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.91, "brier": 0.15047556867808395, "nll": 0.27657795892139925, "ece10": 0.03861290135015798, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.9053792954271751, "confusion_matrix": { "business": { "business": 24, "science and technology": 1, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 5, "science and technology": 17, "sports": 1, "world": 2, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "refund not showing up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "activating a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 47, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47, "brier": 0.8149801405186953, "nll": 1.6501729625826027, "ece10": 0.33269623841793405, "calibration_n": 100, "confident_wrong": 8, "macro_f1": 0.4870970575318401, "confusion_matrix": { "anger": { "anger": 7, "fear": 4, "joy": 0, "love": 0, "sadness": 6, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 0, "fear": 8, "joy": 0, "love": 0, "sadness": 8, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 12, "love": 1, "sadness": 4, "surprise": 0, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 4, "love": 5, "sadness": 8, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 1, "joy": 6, "love": 0, "sadness": 9, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 0, "joy": 5, "love": 0, "sadness": 4, "surprise": 6, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2449, "accuracy": 0.8307327001356852, "wilson95": [ 0.816767908713324, 0.8438366756463843 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2449, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8307327001356852, "brier": 0.25729066675336065, "nll": 0.6154487400537229, "ece10": 0.039838086569453944, "calibration_n": 2948, "confident_wrong": 83, "macro_f1": 0.8270802400791708, "confusion_matrix": { "alarm": { "alarm": 92, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 45, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 9, "iot": 5, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 2, "audio": 0, "calendar": 341, "cooking": 1, "datetime": 21, "email": 0, "general": 4, "iot": 4, "lists": 2, "music": 1, "news": 4, "play": 0, "qa": 1, "recommendation": 15, "social": 1, "takeaway": 2, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 69, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 98, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 1, "email": 236, "general": 17, "iot": 1, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 2, "recommendation": 2, "social": 3, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 10, "cooking": 6, "datetime": 2, "email": 1, "general": 102, "iot": 8, "lists": 0, "music": 4, "news": 2, "play": 5, "qa": 22, "recommendation": 16, "social": 1, "takeaway": 0, "transport": 4, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 3, "calendar": 0, "cooking": 2, "datetime": 0, "email": 0, "general": 3, "iot": 196, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 9, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 1, "datetime": 0, "email": 0, "general": 7, "iot": 4, "lists": 121, "music": 3, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 0, "lists": 0, "music": 73, "news": 0, "play": 2, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 113, "play": 0, "qa": 5, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 8, "calendar": 0, "cooking": 0, "datetime": 2, "email": 2, "general": 12, "iot": 11, "lists": 0, "music": 54, "news": 0, "play": 291, "qa": 0, "recommendation": 5, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 6, "email": 1, "general": 34, "iot": 2, "lists": 0, "music": 3, "news": 4, "play": 1, "qa": 215, "recommendation": 10, "social": 0, "takeaway": 0, "transport": 5, "weather": 5, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 1, "news": 6, "play": 1, "qa": 1, "recommendation": 77, "social": 0, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 3, "general": 14, "iot": 1, "lists": 0, "music": 0, "news": 2, "play": 0, "qa": 0, "recommendation": 0, "social": 85, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 2, "datetime": 2, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 50, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 8, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 6, "social": 0, "takeaway": 0, "transport": 104, "weather": 2, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 4, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 141, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2183, "accuracy": 0.7405020352781547, "wilson95": [ 0.7243723204463967, 0.7560057824367145 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2183, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7405020352781547, "brier": 0.3791136339897723, "nll": 0.9174782305955337, "ece10": 0.06410191223306054, "calibration_n": 2948, "confident_wrong": 78, "macro_f1": 0.7424744363993773, "confusion_matrix": { "alarm": { "alarm": 91, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 2, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 2, "audio": 41, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 11, "iot": 2, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 6, "audio": 0, "calendar": 318, "cooking": 1, "datetime": 30, "email": 2, "general": 11, "iot": 4, "lists": 1, "music": 2, "news": 6, "play": 2, "qa": 1, "recommendation": 9, "social": 2, "takeaway": 2, "transport": 0, "weather": 3, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 64, "datetime": 0, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 1, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 97, "email": 0, "general": 0, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 2, "email": 234, "general": 17, "iot": 2, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 2, "social": 2, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 8, "cooking": 4, "datetime": 7, "email": 1, "general": 105, "iot": 5, "lists": 0, "music": 3, "news": 6, "play": 4, "qa": 13, "recommendation": 14, "social": 1, "takeaway": 3, "transport": 5, "weather": 7, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 4, "calendar": 0, "cooking": 5, "datetime": 1, "email": 1, "general": 10, "iot": 176, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 14, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 1, "calendar": 4, "cooking": 1, "datetime": 0, "email": 0, "general": 10, "iot": 3, "lists": 110, "music": 4, "news": 0, "play": 5, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 4, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 0, "lists": 0, "music": 70, "news": 0, "play": 2, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 1, "email": 1, "general": 8, "iot": 1, "lists": 1, "music": 0, "news": 97, "play": 0, "qa": 7, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 1, "weather": 2, "__failed__": 0 }, "play": { "alarm": 1, "audio": 21, "calendar": 1, "cooking": 0, "datetime": 9, "email": 0, "general": 36, "iot": 7, "lists": 2, "music": 107, "news": 7, "play": 184, "qa": 1, "recommendation": 2, "social": 2, "takeaway": 0, "transport": 1, "weather": 4, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 0, "datetime": 8, "email": 0, "general": 49, "iot": 7, "lists": 0, "music": 1, "news": 11, "play": 1, "qa": 180, "recommendation": 12, "social": 2, "takeaway": 1, "transport": 6, "weather": 7, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 1, "datetime": 1, "email": 0, "general": 6, "iot": 0, "lists": 0, "music": 4, "news": 3, "play": 1, "qa": 1, "recommendation": 66, "social": 0, "takeaway": 1, "transport": 4, "weather": 2, "__failed__": 0 }, "social": { "alarm": 1, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 1, "email": 8, "general": 17, "iot": 1, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 0, "recommendation": 0, "social": 74, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 2, "email": 0, "general": 2, "iot": 1, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 49, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 9, "email": 0, "general": 3, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 8, "social": 0, "takeaway": 1, "transport": 97, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 5, "email": 3, "general": 5, "iot": 3, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 130, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 797, "accuracy": 0.6794543904518329, "wilson95": [ 0.6521988622716927, 0.7055383646457559 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 797, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6794543904518329, "cluster_bootstrap95": [ 0.6511199582081831, 0.7074322655165046 ], "independent_source_groups": 47, "brier": 0.42935575443529295, "nll": 0.7373989302629516, "ece10": 0.03619248639983394, "calibration_n": 1173, "confident_wrong": 3, "macro_f1": 0.6467437356221694, "confusion_matrix": { "negative": { "negative": 104, "neutral": 59, "positive": 19, "__failed__": 0 }, "neutral": { "negative": 90, "neutral": 423, "positive": 77, "__failed__": 0 }, "positive": { "negative": 26, "neutral": 105, "positive": 270, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "nimble-9b": { "label": "Nimble 9B NF4", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 176, "accuracy": 0.88, "wilson95": [ 0.8276574790170326, 0.9180200729362449 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 176, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.1822367249392802, "nll": 0.3489317020925431, "ece10": 0.06747865494432972, "calibration_n": 200, "confident_wrong": 7, "macro_f1": 0.8832914534520471, "confusion_matrix": { "world": { "world": 44, "sports": 0, "business": 2, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 42, "sci_tech": 7, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 6, "sci_tech": 44, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 118, "accuracy": 0.59, "wilson95": [ 0.5207645903369208, 0.6558432509151711 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 118, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.59, "brier": 0.6655747558826097, "nll": 1.5951480173271093, "ece10": 0.27927633694410503, "calibration_n": 200, "confident_wrong": 40, "macro_f1": 0.4850843099372378, "confusion_matrix": { "sadness": { "sadness": 41, "joy": 3, "love": 2, "anger": 5, "fear": 6, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 14, "joy": 48, "love": 9, "anger": 6, "fear": 2, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 1, "joy": 7, "love": 4, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 9, "joy": 3, "love": 0, "anger": 14, "fear": 1, "surprise": 1, "__failed__": 0 }, "fear": { "sadness": 5, "joy": 2, "love": 1, "anger": 1, "fear": 10, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 165, "accuracy": 0.825, "wilson95": [ 0.7663556885175099, 0.8713948493372666 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 165, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.825, "brier": 0.1920655963573828, "nll": 0.29832644752380355, "ece10": 0.06877801347367174, "calibration_n": 200, "confident_wrong": 3, "macro_f1": 0.8117824204780726, "confusion_matrix": { "false": { "false": 109, "true": 0, "__failed__": 0 }, "true": { "false": 35, "true": 56, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 170, "accuracy": 0.85, "wilson95": [ 0.7939442071583334, 0.89286406437758 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 170, "native_tie_decisions_changed": 2, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.21486910917262406, "nll": 0.33687930788806175, "ece10": 0.0676836845592339, "calibration_n": 200, "confident_wrong": 7, "macro_f1": 0.830393487109905, "confusion_matrix": { "true": { "true": 51, "false": 26, "__failed__": 0 }, "false": { "true": 4, "false": 119, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.23950363241719094, "nll": 0.48270231704190286, "ece10": 0.13547132685976818, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8666666666666666, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.21561822020985358, "nll": 0.49153415748644075, "ece10": 0.1486410212359033, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8533333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03704976355906967, "nll": 0.06283071834612493, "ece10": 0.04896050005849013, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.07645215681321074, "nll": 0.10527876151470598, "ece10": 0.06452838833289638, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9142857142857143, "confusion_matrix": { "design": { "design": 2, "inspection": 0, "purchase": 1, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 94, "accuracy": 0.94, "wilson95": [ 0.8752318455410406, 0.9722138760368118 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 94, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.94, "brier": 0.10161924818545379, "nll": 0.21588406841713834, "ece10": 0.019063207754146654, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.9393627450980392, "confusion_matrix": { "label_000": { "label_000": 22, "label_001": 3, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 22, "label_002": 1, "label_003": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 45, "accuracy": 0.45, "wilson95": [ 0.3561453797951198, 0.5475539700255787 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 45, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.8268843824349762, "nll": 1.868259208373307, "ece10": 0.3434679904627446, "calibration_n": 100, "confident_wrong": 16, "macro_f1": 0.4431908821522525, "confusion_matrix": { "label_000": { "label_000": 7, "label_001": 2, "label_002": 1, "label_003": 1, "label_004": 5, "label_005": 1, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 11, "label_002": 1, "label_003": 0, "label_004": 3, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 11, "label_003": 1, "label_004": 3, "label_005": 1, "__failed__": 0 }, "label_003": { "label_000": 1, "label_001": 0, "label_002": 5, "label_003": 5, "label_004": 6, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 1, "label_001": 2, "label_002": 2, "label_003": 2, "label_004": 8, "label_005": 1, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 1, "label_002": 5, "label_003": 0, "label_004": 6, "label_005": 3, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 84, "accuracy": 0.84, "wilson95": [ 0.7557973061072976, 0.8990471151119521 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 84, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.84, "brier": 0.22321636863385305, "nll": 0.5573742691498392, "ece10": 0.057056982335578856, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.8342592592592593, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 2, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 1, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 2, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 1, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 2, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 3.856342975599653e-06, "nll": 0.0009950695347858151, "ece10": 0.0009938777012235209, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 65, "accuracy": 0.5855855855855856, "wilson95": [ 0.4925800538625369, 0.672865423712468 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 65, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5855855855855856, "brier": 0.564514755463128, "nll": 1.0268878342963967, "ece10": 0.19882437128226776, "calibration_n": 111, "confident_wrong": 9 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 68, "accuracy": 0.9444444444444444, "wilson95": [ 0.8656798403400208, 0.9781857840186834 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 68, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "cluster_bootstrap95": [ 0.8611111111111112, 1.0 ], "independent_source_groups": 36, "brier": 0.09523220192785822, "nll": 0.18166928300371268, "ece10": 0.04702223790802408, "calibration_n": 72, "confident_wrong": 3 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 17, "accuracy": 0.8947368421052632, "wilson95": [ 0.6860591728889099, 0.9706414397011299 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8947368421052632, "brier": 0.2003918763092868, "nll": 0.355848839163076, "ece10": 0.10167184109062138, "calibration_n": 19, "confident_wrong": 1, "macro_f1": 0.8944444444444444, "confusion_matrix": { "false": { "false": 9, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.059580228531038074, "nll": 0.1110907516883979, "ece10": 0.08740745649883391, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9372549019607843, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 18, "accuracy": 0.8181818181818182, "wilson95": [ 0.6148339256287345, 0.9269311541657794 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8181818181818182, "brier": 0.2704816633525533, "nll": 0.5591910994361969, "ece10": 0.11320099117978882, "calibration_n": 22, "confident_wrong": 2, "macro_f1": 0.8035714285714286, "confusion_matrix": { "false": { "false": 6, "true": 4, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00019001357396357884, "nll": 0.007135600105517963, "ece10": 0.007078956969945871, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06473166490178967, "nll": 0.13329620866889683, "ece10": 0.107172290973163, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.030088266719018496, "nll": 0.04757793172247022, "ece10": 0.03542776114257076, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 1, "roads": 3, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.07807494921877317, "nll": 0.14661355541798055, "ece10": 0.04153496148088165, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9206349206349206, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 3, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 13, "accuracy": 0.8666666666666667, "wilson95": [ 0.6211801716767764, 0.9626387408969346 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8666666666666667, "brier": 0.13635782392835571, "nll": 0.3773778141769219, "ece10": 0.11186566076818229, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.8539682539682539, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 1, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 2, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 1, "docs": 0, "test": 0, "chore": 3, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.26483439870460845, "nll": 0.41354603376657134, "ece10": 0.13408571840604497, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.8131868131868132, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.23636974132781632, "nll": 0.494019824411127, "ece10": 0.10028733605329959, "calibration_n": 16, "confident_wrong": 2, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.23988875646336455, "nll": 0.4708174832152003, "ece10": 0.13686423746243082, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.7933333333333333, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 1, "threat": 1, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 5, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 21, "accuracy": 0.9545454545454546, "wilson95": [ 0.7820195416134267, 0.991930572378736 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9545454545454546, "brier": 0.1552209956633763, "nll": 0.337903540257247, "ece10": 0.11451941650815389, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 1, "feature": 3, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 7.127599409633721e-05, "nll": 0.004303313881130817, "ece10": 0.004282536786162214, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 7.140215729191767e-05, "nll": 0.00453945661787204, "ece10": 0.004512146236081316, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.009162403898791277, "nll": 0.04012895445466618, "ece10": 0.0375560092916662, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0030729222929600534, "nll": 0.019307350711288904, "ece10": 0.018307161184581318, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.11025784938399005, "nll": 0.1722512064483047, "ece10": 0.0531422113222383, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.9314285714285713, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.009084753844793838, "nll": 0.03440111278848505, "ece10": 0.03151301030456087, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 3, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.13853524651766166, "nll": 0.21275898873986238, "ece10": 0.07487824907193073, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8888888888888888, "confusion_matrix": { "false": { "false": 8, "true": 1, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.494188313436166, "nll": 0.6903316116636872, "ece10": 0.270930934923087, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6363636363636362, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 5, "3": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.10899306173441398, "nll": 0.23375223399140108, "ece10": 0.1925650459870061, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 18, "accuracy": 0.9, "wilson95": [ 0.6989663547715128, 0.9721335187862319 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.1674241225610265, "nll": 0.29147791731453104, "ece10": 0.13154302458079628, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9095238095238095, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 3, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 8, "accuracy": 0.38095238095238093, "wilson95": [ 0.20751011368564493, 0.5912134622488425 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.38095238095238093, "brier": 1.0938218316169024, "nll": 2.807543435878128, "ece10": 0.4434446380904019, "calibration_n": 21, "confident_wrong": 8, "macro_f1": 0.3269565217391305, "confusion_matrix": { "spelling": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 4, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 0, "punctuation": 0, "none": 1, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 0, "none": 3, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02223448889028206, "nll": 0.048130419327717326, "ece10": 0.04049891492546886, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0016115452464493653, "nll": 0.010862899733469019, "ece10": 0.010408457999316681, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 11, "accuracy": 0.7857142857142857, "wilson95": [ 0.524107694133997, 0.9242861328730683 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.291735062167409, "nll": 0.5309610888512063, "ece10": 0.15411732013135707, "calibration_n": 14, "confident_wrong": 1, "macro_f1": 0.6961038961038961, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 2, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.13730099478946317, "nll": 0.2951151042189885, "ece10": 0.09676679496193688, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9444444444444444, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.3900875836486402, "nll": 0.6401240433245874, "ece10": 0.10254705368362123, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6976190476190476, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 2, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.2789641603171635, "nll": 0.5195858396349881, "ece10": 0.15745711640226517, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.7571428571428571, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02156250556443818, "nll": 0.057381326494054134, "ece10": 0.04867317292460431, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.11289461577937761, "nll": 0.17416862969846456, "ece10": 0.08115691595454821, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9028571428571428, "confusion_matrix": { "payments": { "payments": 3, "web": 0, "infra": 1, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 3, "infra": 1, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.1473724761869537, "nll": 0.27052181094652694, "ece10": 0.21640625187223855, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8819444444444444, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.08857948145205428, "nll": 0.13010942347053805, "ece10": 0.06918397200322911, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9372549019607843, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.3023729193955404, "nll": 0.4673847856468681, "ece10": 0.21121459246943322, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8119658119658119, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 3, "2": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.038721465545057385, "nll": 0.06428945071822396, "ece10": 0.03753586608994416, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571427, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.1751909889113056, "nll": 0.27995323208417877, "ece10": 0.10915231924662548, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8819444444444444, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0008378241336360064, "nll": 0.01659519614030211, "ece10": 0.01631401878552315, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 4, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.5023913626618203, "nll": 1.2865032996032921, "ece10": 0.18257584127442666, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.6533333333333333, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 2, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 2, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.6511707913656924, "nll": 1.0861222898327072, "ece10": 0.37375835182564665, "calibration_n": 16, "confident_wrong": 4, "macro_f1": 0.6190476190476191, "confusion_matrix": { "false": { "false": 4, "true": 6, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 12, "accuracy": 0.8, "wilson95": [ 0.5481455128483065, 0.9295245065301844 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.3210342389349729, "nll": 0.4745580072501686, "ece10": 0.18462278129652263, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.7963800904977376, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.09558231501611152, "nll": 0.16144652640117113, "ece10": 0.1173250169061004, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 4, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.032059275295996136, "nll": 0.09127948748942215, "ece10": 0.08060576399199088, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 16, "accuracy": 0.8421052631578947, "wilson95": [ 0.6243452472967093, 0.9447952836146585 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8421052631578947, "brier": 0.18438599411612105, "nll": 0.3465345813675554, "ece10": 0.1996789599305058, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8122086247086246, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 2, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.16040780612470779, "nll": 0.28127381823414355, "ece10": 0.14637105874649578, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8712121212121212, "confusion_matrix": { "false": { "false": 10, "true": 1, "__failed__": 0 }, "true": { "false": 1, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.004889987757837602, "nll": 0.02683972429355797, "ece10": 0.025475955484582164, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 16, "accuracy": 0.8, "wilson95": [ 0.5839825677481064, 0.919342337420202 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.2904036029105679, "nll": 0.46817419555535833, "ece10": 0.15017539455422482, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7998737373737373, "confusion_matrix": { "0": { "0": 5, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.058831558462799735, "nll": 0.08518720436896289, "ece10": 0.044002179252058, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 1, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 3, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.0357233520028781, "nll": 0.07020875381253412, "ece10": 0.05589551962651744, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.949874686716792, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.07271843610870767, "nll": 0.14800245607048257, "ece10": 0.10500921830443014, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9113247863247864, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 584, "accuracy": 0.6488888888888888, "wilson95": [ 0.6171318176276369, 0.6793803609016357 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 745, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6943150046598322, "cluster_bootstrap95": [ 0.6087704076621753, 0.6976509850307139 ], "independent_source_groups": 596, "brier": 0.4203160733617292, "nll": 0.8487739030788213, "ece10": 0.12760640245540486, "calibration_n": 1073, "confident_wrong": 63 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 398, "accuracy": 0.6933797909407665, "wilson95": [ 0.6544772704663733, 0.7297111545958845 ], "valid": 573, "failures": 1, "coverage": 0.9982578397212544, "heads": 936, "head_correct": 748, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7991452991452992, "brier": 0.2987691240436231, "nll": 0.6325609877247241, "ece10": 0.08134221832380158, "calibration_n": 934, "confident_wrong": 50 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 316, "accuracy": 0.4514285714285714, "wilson95": [ 0.41492879364937185, 0.4884585400352113 ], "valid": 649, "failures": 51, "coverage": 0.9271428571428572, "heads": 1088, "head_correct": 593, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5450367647058824, "cluster_bootstrap95": [ 0.4176108851245598, 0.4878057503751003 ], "independent_source_groups": 650, "brier": 0.5622794659056807, "nll": 1.1631376759915446, "ece10": 0.16228344004738304, "calibration_n": 986, "confident_wrong": 66 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 74, "accuracy": 0.185, "wilson95": [ 0.15000697382952777, 0.22598577160107902 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1355, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6775, "brier": 0.4330990514276924, "nll": 0.7939870301163368, "ece10": 0.050448052153739424, "calibration_n": 2000, "confident_wrong": 52 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.004725791557016422, "nll": 0.024000484426497073, "ece10": 0.022543372334066815, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.007758115276604235, "nll": 0.024237592694670636, "ece10": 0.021738043761915873, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.010866164236259237, "nll": 0.03999131329002322, "ece10": 0.036310505614258665, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.011154948924152506, "nll": 0.03871149129299534, "ece10": 0.035127814072852266, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 39, "accuracy": 0.975, "wilson95": [ 0.8711863103652591, 0.9955731684973186 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.975, "brier": 0.024652362494883874, "nll": 0.066946480235385, "ece10": 0.05323354339471087, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9749019607843138, "confusion_matrix": { "review": { "review": 7, "it": 0, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 39, "accuracy": 0.975, "wilson95": [ 0.8711863103652591, 0.9955731684973186 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.975, "brier": 0.025996203038651577, "nll": 0.06893099404668511, "ece10": 0.04606291670666207, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9749019607843138, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04302484833425387, "nll": 0.11887389197489688, "ece10": 0.10003740398640225, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "review": { "review": 8, "it": 0, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 39, "accuracy": 0.975, "wilson95": [ 0.8711863103652591, 0.9955731684973186 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.975, "brier": 0.06150811670558387, "nll": 0.14487480607668254, "ece10": 0.07086134676763547, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9749019607843138, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 234, "accuracy": 0.585, "wilson95": [ 0.5361289851942794, 0.6322539247688901 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 234, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.585, "brier": 0.6275188674323338, "nll": 1.4315864407161305, "ece10": 0.24661810474891838, "calibration_n": 400, "confident_wrong": 51, "macro_f1": 0.47600151984232975, "confusion_matrix": { "sadness": { "sadness": 83, "joy": 15, "love": 7, "anger": 3, "fear": 10, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 18, "joy": 76, "love": 14, "anger": 4, "fear": 3, "surprise": 3, "__failed__": 0 }, "love": { "sadness": 8, "joy": 12, "love": 10, "anger": 4, "fear": 2, "surprise": 1, "__failed__": 0 }, "anger": { "sadness": 16, "joy": 7, "love": 3, "anger": 33, "fear": 2, "surprise": 3, "__failed__": 0 }, "fear": { "sadness": 10, "joy": 4, "love": 1, "anger": 5, "fear": 31, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 3, "love": 1, "anger": 1, "fear": 2, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 278, "accuracy": 0.9266666666666666, "wilson95": [ 0.8914686583942826, 0.9510760061607713 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 278, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9266666666666666, "brier": 0.11639261214986783, "nll": 0.30960907807200827, "ece10": 0.028839898561458254, "calibration_n": 300, "confident_wrong": 3 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 342, "accuracy": 0.855, "wilson95": [ 0.8171167606818923, 0.8861295103407572 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 342, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.855, "brier": 0.2050946331715246, "nll": 0.3176398857387119, "ece10": 0.03158509619928409, "calibration_n": 400, "confident_wrong": 11, "macro_f1": 0.8138878192786548, "confusion_matrix": { "false": { "false": 265, "true": 44, "__failed__": 0 }, "true": { "false": 14, "true": 77, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 354, "accuracy": 0.885, "wilson95": [ 0.850008455984224, 0.9126670773783676 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 354, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.885, "brier": 0.1845894078934991, "nll": 0.39441713236970877, "ece10": 0.068871698946846, "calibration_n": 400, "confident_wrong": 20, "macro_f1": 0.8765700498514106, "confusion_matrix": { "world": { "world": 88, "sports": 6, "business": 9, "sci_tech": 0, "__failed__": 0 }, "sports": { "world": 3, "sports": 120, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 3, "sports": 0, "business": 66, "sci_tech": 3, "__failed__": 0 }, "sci_tech": { "world": 7, "sports": 0, "business": 15, "sci_tech": 80, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 337, "accuracy": 0.8425, "wilson95": [ 0.8035652075632727, 0.8749188708177342 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 337, "native_tie_decisions_changed": 3, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8425, "brier": 0.19574540197926624, "nll": 0.2964239593478713, "ece10": 0.06350266405596838, "calibration_n": 400, "confident_wrong": 10, "macro_f1": 0.8171249428515446, "confusion_matrix": { "true": { "true": 94, "false": 59, "__failed__": 0 }, "false": { "true": 4, "false": 243, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 240, "accuracy": 0.8, "wilson95": [ 0.7510710762322137, 0.8413431410330587 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 240, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.30745633576701326, "nll": 0.7291116449226411, "ece10": 0.11139496689233666, "calibration_n": 300, "confident_wrong": 19 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 337, "accuracy": 0.8425, "wilson95": [ 0.8035652075632727, 0.8749188708177342 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 337, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8425, "brier": 0.1885734833094562, "nll": 0.28769312880069203, "ece10": 0.03455393537569725, "calibration_n": 400, "confident_wrong": 5, "macro_f1": 0.8409883959388942, "confusion_matrix": { "false": { "false": 188, "true": 0, "__failed__": 0 }, "true": { "false": 63, "true": 149, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 341, "accuracy": 0.8525, "wilson95": [ 0.8143999058527144, 0.8838939266416067 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 341, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8525, "brier": 0.21575372310034552, "nll": 0.35364058990523645, "ece10": 0.03473236055141528, "calibration_n": 400, "confident_wrong": 11, "macro_f1": 0.8518248770962699, "confusion_matrix": { "false": { "false": 184, "true": 16, "__failed__": 0 }, "true": { "false": 43, "true": 157, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 73, "accuracy": 0.73, "wilson95": [ 0.6356788323205499, 0.8073041585042365 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 73, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.73, "brier": 0.41791739388001375, "nll": 0.7090528125810543, "ece10": 0.20129259600136298, "calibration_n": 100, "confident_wrong": 14, "macro_f1": 0.7149192271143491, "confusion_matrix": { "yes": { "yes": 48, "no": 6, "__failed__": 0 }, "no": { "yes": 21, "no": 25, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 52, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.7658260643565075, "nll": 1.9644873699129946, "ece10": 0.27225928255712706, "calibration_n": 100, "confident_wrong": 18, "macro_f1": 0.549118966688626, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 4, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 5, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 4, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 4, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 6, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 5, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 4, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 4, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 2, "limit_increase": 0, "other": 1, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 144, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.72, "brier": 0.3958289006313284, "nll": 0.699879182897208, "ece10": 0.1667481041040097, "calibration_n": 200, "confident_wrong": 16 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 127, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.635, "brier": 0.5584406354502591, "nll": 1.3252932554809758, "ece10": 0.22538994342404323, "calibration_n": 200, "confident_wrong": 25 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 79, "accuracy": 0.79, "wilson95": [ 0.7002003116591013, 0.858343459380847 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 79, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.34112094210138194, "nll": 1.0852565254436695, "ece10": 0.14604254361446464, "calibration_n": 100, "confident_wrong": 10, "macro_f1": 0.7634532283956782, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 14, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 1, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 12, "clinical_note": 0, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 3, "resume": 0, "bank_statement": 0, "clinical_note": 5, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 2, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 5, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 6, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 1, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 9, "accuracy": 0.09, "wilson95": [ 0.04807254000256516, 0.1622621285271631 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 220, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.7176942793265245, "nll": 1.7104493294176248, "ece10": 0.2516239274735882, "calibration_n": 400, "confident_wrong": 63 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 62, "accuracy": 0.62, "wilson95": [ 0.5220975529551107, 0.7090240074752127 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 62, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.62, "brier": 0.5550889838637978, "nll": 1.5755796682109002, "ece10": 0.2185252529193619, "calibration_n": 100, "confident_wrong": 9, "macro_f1": 0.6038961038961039, "confusion_matrix": { "politics": { "politics": 0, "business": 1, "technology": 2, "science": 3, "health": 3, "sports": 0, "entertainment": 0, "world": 3, "culture": 1, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 1, "technology": 8, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 1, "technology": 1, "science": 5, "health": 0, "sports": 0, "entertainment": 0, "world": 2, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 6, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 6, "world": 0, "culture": 2, "weather": 0, "__failed__": 0 }, "world": { "politics": 0, "business": 2, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 5, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 2, "world": 0, "culture": 6, "weather": 1, "__failed__": 0 }, "weather": { "politics": 0, "business": 2, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 7, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 40, "accuracy": 0.4, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.9085377336478146, "nll": 2.997217452591172, "ece10": 0.3461310789389606, "calibration_n": 100, "confident_wrong": 10, "macro_f1": 0.37461032800216865, "confusion_matrix": { "biology": { "biology": 5, "medicine": 1, "chemistry": 1, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 9, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 5, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 3, "medicine": 2, "chemistry": 1, "physics": 4, "computer_science": 0, "mathematics": 0, "earth_science": 3, "engineering": 2, "economics": 1, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 3, "chemistry": 2, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 3, "engineering": 5, "economics": 1, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 1, "chemistry": 1, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 1, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 1, "medicine": 3, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 6, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 5, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 4, "other": 0, "__failed__": 0 }, "other": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 1, "mathematics": 1, "earth_science": 3, "engineering": 6, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 67, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.335, "brier": 0.624458154393168, "nll": 1.4199471619773691, "ece10": 0.23974413587086058, "calibration_n": 100, "confident_wrong": 14 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 35, "accuracy": 0.35, "wilson95": [ 0.26364248243097527, 0.44745556703112027 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 122, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.3321705322242774, "nll": 0.6353071651044264, "ece10": 0.1292412557773457, "calibration_n": 100, "confident_wrong": 8 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 84, "accuracy": 0.84, "wilson95": [ 0.7557973061072976, 0.8990471151119521 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 84, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.84, "brier": 0.23993816431103818, "nll": 0.4660096558839635, "ece10": 0.08733134997999242, "calibration_n": 100, "confident_wrong": 8, "macro_f1": 0.8357142857142857, "confusion_matrix": { "negative": { "negative": 36, "neutral": 0, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 10, "neutral": 21, "positive": 2, "__failed__": 0 }, "positive": { "negative": 2, "neutral": 2, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 3, "accuracy": 0.03, "wilson95": [ 0.010254524024038925, 0.0845193642905276 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.2845352907072238, "nll": 0.4291810311783179, "ece10": 0.08275957347472121, "calibration_n": 100, "confident_wrong": 2 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 228, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.34382602259504175, "nll": 0.8351258198335056, "ece10": 0.13138759684156268, "calibration_n": 300, "confident_wrong": 23 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 70, "accuracy": 0.7, "wilson95": [ 0.6041514536665332, 0.7810511470506724 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 70, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.46104950153174806, "nll": 1.4943386063343025, "ece10": 0.09037869217142577, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.6733140225787285, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 3, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 2, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 5, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 2, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 1, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 4, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 3, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 4, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 2, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 2, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 5, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 5, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 2, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 3, "other": 1, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 39, "accuracy": 0.39, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.39, "brier": 0.8846268833120915, "nll": 2.2082328566147167, "ece10": 0.36858116188875834, "calibration_n": 100, "confident_wrong": 12, "macro_f1": 0.3785909259822304, "confusion_matrix": { "billing": { "billing": 3, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 2, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 6, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 4, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 1, "onboarding": 0, "integrations": 4, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 1, "onboarding": 2, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 7, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "compliance": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 1, "pricing": 1, "data_privacy": 1, "other": 1, "__failed__": 0 }, "pricing": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 3, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 2, "compliance": 0, "pricing": 1, "data_privacy": 2, "other": 1, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 7, "accuracy": 0.07, "wilson95": [ 0.03431926106727268, 0.137495147390735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 143, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4766666666666667, "brier": 0.7540668040913054, "nll": 1.9194392051083142, "ece10": 0.28404071548405785, "calibration_n": 300, "confident_wrong": 27 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 60, "accuracy": 0.6, "wilson95": [ 0.5020025867910618, 0.6905987135675411 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 60, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.6474561170068843, "nll": 1.6978313189005336, "ece10": 0.27692236996816894, "calibration_n": 100, "confident_wrong": 17, "macro_f1": 0.5819747089483932, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 4, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 2, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 2, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 11, "refund": 0, "hotel": 0, "car": 2, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 3, "loyalty": 0, "disruption": 0, "other": 2, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 1, "loyalty": 6, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 5, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 4, "car": 2, "visa_docs": 1, "loyalty": 1, "disruption": 1, "other": 1, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 95, "accuracy": 0.95, "wilson95": [ 0.8882495307680808, 0.9784563208456319 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 95, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.0850496864506795, "nll": 0.1882008370173844, "ece10": 0.03455275645084763, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.9497549019607843, "confusion_matrix": { "business": { "business": 22, "science and technology": 3, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 1, "science and technology": 23, "sports": 1, "world": 0, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 85, "accuracy": 0.85, "wilson95": [ 0.7671644040916763, 0.9069401471634337 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 85, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.19031493324866172, "nll": 0.44075978188843584, "ece10": 0.09628960496034382, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8462962962962962, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 2, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 2, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 1, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 1, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 45, "accuracy": 0.45, "wilson95": [ 0.3561453797951198, 0.5475539700255787 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 45, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.8484351020870358, "nll": 2.048481049708925, "ece10": 0.3654884450455158, "calibration_n": 100, "confident_wrong": 18, "macro_f1": 0.42439168819982775, "confusion_matrix": { "anger": { "anger": 9, "fear": 3, "joy": 0, "love": 1, "sadness": 4, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 1, "fear": 12, "joy": 1, "love": 0, "sadness": 3, "surprise": 0, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 11, "love": 2, "sadness": 2, "surprise": 1, "__failed__": 0 }, "love": { "anger": 1, "fear": 1, "joy": 6, "love": 3, "sadness": 6, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 1, "fear": 2, "joy": 3, "love": 1, "sadness": 8, "surprise": 1, "__failed__": 0 }, "surprise": { "anger": 2, "fear": 1, "joy": 5, "love": 0, "sadness": 6, "surprise": 2, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2411, "accuracy": 0.817842605156038, "wilson95": [ 0.8034989632345869, 0.8313589809141791 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2411, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.817842605156038, "brier": 0.27313569271982907, "nll": 0.6592560098970066, "ece10": 0.08210357512425642, "calibration_n": 2948, "confident_wrong": 152, "macro_f1": 0.826358763809085, "confusion_matrix": { "alarm": { "alarm": 94, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 47, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 8, "iot": 4, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 5, "audio": 0, "calendar": 358, "cooking": 0, "datetime": 9, "email": 0, "general": 8, "iot": 0, "lists": 2, "music": 0, "news": 2, "play": 1, "qa": 4, "recommendation": 9, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 67, "datetime": 0, "email": 0, "general": 0, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 92, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 6, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 1, "calendar": 6, "cooking": 0, "datetime": 0, "email": 216, "general": 41, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "general": { "alarm": 1, "audio": 0, "calendar": 7, "cooking": 6, "datetime": 0, "email": 0, "general": 123, "iot": 2, "lists": 0, "music": 1, "news": 1, "play": 3, "qa": 29, "recommendation": 9, "social": 1, "takeaway": 0, "transport": 2, "weather": 2, "__failed__": 0 }, "iot": { "alarm": 2, "audio": 2, "calendar": 1, "cooking": 3, "datetime": 0, "email": 0, "general": 5, "iot": 199, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 1, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 0, "email": 0, "general": 8, "iot": 3, "lists": 121, "music": 3, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 0, "lists": 1, "music": 70, "news": 0, "play": 2, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 3, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 9, "iot": 0, "lists": 0, "music": 0, "news": 96, "play": 0, "qa": 14, "recommendation": 0, "social": 1, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 6, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 35, "iot": 6, "lists": 1, "music": 89, "news": 1, "play": 244, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "qa": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 35, "iot": 4, "lists": 0, "music": 3, "news": 2, "play": 1, "qa": 234, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 2, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 0, "news": 2, "play": 1, "qa": 1, "recommendation": 82, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 36, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 66, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 46, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 5, "social": 0, "takeaway": 0, "transport": 114, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 3, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 142, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2181, "accuracy": 0.7398236092265943, "wilson95": [ 0.7236813945294366, 0.7553416220267827 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2181, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7398236092265943, "brier": 0.38265704668351896, "nll": 0.9464729969797974, "ece10": 0.1204796153823034, "calibration_n": 2948, "confident_wrong": 176, "macro_f1": 0.7549808819449209, "confusion_matrix": { "alarm": { "alarm": 93, "audio": 2, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 0, "audio": 34, "calendar": 1, "cooking": 0, "datetime": 4, "email": 0, "general": 12, "iot": 9, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 9, "audio": 1, "calendar": 326, "cooking": 0, "datetime": 8, "email": 1, "general": 26, "iot": 4, "lists": 2, "music": 0, "news": 0, "play": 1, "qa": 4, "recommendation": 13, "social": 2, "takeaway": 1, "transport": 1, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 65, "datetime": 0, "email": 0, "general": 1, "iot": 1, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 97, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 1, "email": 213, "general": 42, "iot": 0, "lists": 2, "music": 1, "news": 0, "play": 0, "qa": 2, "recommendation": 1, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 6, "cooking": 5, "datetime": 1, "email": 1, "general": 130, "iot": 2, "lists": 1, "music": 2, "news": 1, "play": 2, "qa": 20, "recommendation": 9, "social": 0, "takeaway": 0, "transport": 3, "weather": 3, "__failed__": 0 }, "iot": { "alarm": 2, "audio": 4, "calendar": 0, "cooking": 2, "datetime": 1, "email": 0, "general": 12, "iot": 190, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 1, "calendar": 2, "cooking": 1, "datetime": 0, "email": 0, "general": 10, "iot": 3, "lists": 122, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 12, "iot": 1, "lists": 3, "music": 59, "news": 0, "play": 3, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 3, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 22, "iot": 1, "lists": 1, "music": 0, "news": 80, "play": 1, "qa": 10, "recommendation": 2, "social": 2, "takeaway": 0, "transport": 0, "weather": 2, "__failed__": 0 }, "play": { "alarm": 1, "audio": 7, "calendar": 1, "cooking": 1, "datetime": 5, "email": 0, "general": 81, "iot": 18, "lists": 9, "music": 108, "news": 0, "play": 150, "qa": 2, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 1, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 2, "email": 0, "general": 54, "iot": 4, "lists": 1, "music": 3, "news": 2, "play": 1, "qa": 212, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 2, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 2, "qa": 3, "recommendation": 78, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 45, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 53, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 2, "datetime": 1, "email": 0, "general": 12, "iot": 2, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 36, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 5, "social": 0, "takeaway": 0, "transport": 108, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 8, "iot": 4, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 135, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 636, "accuracy": 0.5421994884910486, "wilson95": [ 0.5135966947891232, 0.5705267861249562 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 636, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5421994884910486, "cluster_bootstrap95": [ 0.49403795167556336, 0.586128282312468 ], "independent_source_groups": 47, "brier": 0.6782912373777158, "nll": 1.2657678201425693, "ece10": 0.2715904431528394, "calibration_n": 1173, "confident_wrong": 161, "macro_f1": 0.5305225260557137, "confusion_matrix": { "negative": { "negative": 121, "neutral": 18, "positive": 43, "__failed__": 0 }, "neutral": { "negative": 107, "neutral": 165, "positive": 318, "__failed__": 0 }, "positive": { "negative": 35, "neutral": 16, "positive": 350, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "plumb-4b": { "label": "Plumb 4B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 171, "accuracy": 0.855, "wilson95": [ 0.7995133246921847, 0.8971064935799559 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 171, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.855, "brier": 0.19427349388490314, "nll": 0.3786339639189171, "ece10": 0.03706386330442153, "calibration_n": 200, "confident_wrong": 3, "macro_f1": 0.8589443054009266, "confusion_matrix": { "world": { "world": 43, "sports": 0, "business": 3, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 45, "business": 2, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 3, "sports": 0, "business": 45, "sci_tech": 3, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 12, "sci_tech": 38, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 116, "accuracy": 0.58, "wilson95": [ 0.5107207393958046, 0.6462640083838327 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 116, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.58, "brier": 0.6145615279397716, "nll": 1.2591749138128188, "ece10": 0.15084603829829216, "calibration_n": 200, "confident_wrong": 6, "macro_f1": 0.48914915070625803, "confusion_matrix": { "sadness": { "sadness": 46, "joy": 2, "love": 1, "anger": 3, "fear": 5, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 19, "joy": 42, "love": 11, "anger": 2, "fear": 5, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 2, "joy": 5, "love": 5, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 10, "joy": 3, "love": 2, "anger": 12, "fear": 1, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 5, "joy": 2, "love": 1, "anger": 1, "fear": 10, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 184, "accuracy": 0.92, "wilson95": [ 0.874010511732856, 0.9501594141102401 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 184, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.92, "brier": 0.1239240016629439, "nll": 0.22403504819755382, "ece10": 0.06279215551893919, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.9176191947276284, "confusion_matrix": { "false": { "false": 109, "true": 0, "__failed__": 0 }, "true": { "false": 16, "true": 75, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 183, "accuracy": 0.915, "wilson95": [ 0.8681041292843443, 0.9462542498225244 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 183, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.915, "brier": 0.1368688058252287, "nll": 0.23576317983830578, "ece10": 0.0292822727259867, "calibration_n": 200, "confident_wrong": 5, "macro_f1": 0.9056368127445811, "confusion_matrix": { "true": { "true": 60, "false": 17, "__failed__": 0 }, "false": { "true": 0, "false": 123, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.29684732642777273, "nll": 0.5647088480679607, "ece10": 0.200168170276088, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.7676190476190475, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3465980335321934, "nll": 0.7808621906541036, "ece10": 0.19985117996033752, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.76, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 1, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03974780537694105, "nll": 0.1439604114823919, "ece10": 0.12867084952381241, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0142694268457341, "nll": 0.08363616910177195, "ece10": 0.07838564901326141, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 89, "accuracy": 0.89, "wilson95": [ 0.8136870349691969, 0.9374580364293543 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 89, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.89, "brier": 0.17050518994493624, "nll": 0.3370007652073042, "ece10": 0.1261028014802225, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8804870754667915, "confusion_matrix": { "label_000": { "label_000": 25, "label_001": 0, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 8, "label_001": 14, "label_002": 1, "label_003": 2, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 47, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47, "brier": 0.6883173471889722, "nll": 1.4181417680716757, "ece10": 0.17751330564463863, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.47823857204094294, "confusion_matrix": { "label_000": { "label_000": 9, "label_001": 3, "label_002": 1, "label_003": 1, "label_004": 3, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 10, "label_002": 1, "label_003": 0, "label_004": 4, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 9, "label_003": 1, "label_004": 4, "label_005": 2, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 3, "label_003": 6, "label_004": 8, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 2, "label_002": 2, "label_003": 3, "label_004": 8, "label_005": 1, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 1, "label_002": 5, "label_003": 0, "label_004": 4, "label_005": 5, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 77, "accuracy": 0.77, "wilson95": [ 0.6784561697712622, 0.8415673411969654 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 77, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.77, "brier": 0.3457448184028046, "nll": 0.9003317047307803, "ece10": 0.08702095162910585, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.7330026455026455, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 2, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 1, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 2, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 1, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 1, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 1, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0066428372190285466, "nll": 0.05771959515637981, "ece10": 0.05543789281564592, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 89, "accuracy": 0.8018018018018018, "wilson95": [ 0.7181017378609362, 0.8653112620088181 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 89, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8018018018018018, "brier": 0.33486152971777744, "nll": 0.6511237978744568, "ece10": 0.09263801516578161, "calibration_n": 111, "confident_wrong": 6 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 70, "accuracy": 0.9722222222222222, "wilson95": [ 0.9042582477856177, 0.9923489780955054 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 70, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9722222222222222, "cluster_bootstrap95": [ 0.9305555555555556, 1.0 ], "independent_source_groups": 36, "brier": 0.08336187050616783, "nll": 0.21152344655884975, "ece10": 0.15312195418375993, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.08837776671259015, "nll": 0.19106854336800724, "ece10": 0.11162424908906968, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9449275362318841, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03422790598264689, "nll": 0.12127221602611744, "ece10": 0.1114923938954892, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.2565915646941674, "nll": 0.4123109834854544, "ece10": 0.17685757699219778, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8562091503267973, "confusion_matrix": { "false": { "false": 7, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.06345201836101443, "nll": 0.1433274348326849, "ece10": 0.10698290823625921, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9415584415584416, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 1, "feature_request": 0, "praise": 0, "pricing_billing": 3, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.16364878903017469, "nll": 0.37583362452500574, "ece10": 0.25900680301198176, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8825396825396826, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.03512578654700702, "nll": 0.11032216099530345, "ece10": 0.09318185602275902, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 1, "roads": 3, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.19544359779840997, "nll": 0.4209984405220006, "ece10": 0.176190793043201, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8444444444444444, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 1, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 12, "accuracy": 0.8, "wilson95": [ 0.5481455128483065, 0.9295245065301844 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.26149780215330853, "nll": 0.5590488659774309, "ece10": 0.2828395455688434, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7888888888888889, "confusion_matrix": { "feature": { "feature": 2, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 1, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 1, "test": 0, "chore": 1, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.05383589347439222, "nll": 0.15771878653973007, "ece10": 0.1416218194868595, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.2107615820558327, "nll": 0.372250712331921, "ece10": 0.12045073983896257, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.19603463603675356, "nll": 0.4051788051600803, "ece10": 0.19040624561723118, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8377777777777778, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 1, "threat": 1, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 1, "none": 5, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.22126983914567863, "nll": 0.4574763704323551, "ece10": 0.16438704139742819, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8565656565656565, "confusion_matrix": { "news_report": { "news_report": 4, "opinion": 0, "press_release": 1, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 2, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 2, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00827101776827629, "nll": 0.06670225544670982, "ece10": 0.0634348215214009, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.005669026054649845, "nll": 0.060190507307616783, "ece10": 0.057780124538776456, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03876810811740851, "nll": 0.14130896787361355, "ece10": 0.13035513454727168, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.009474013416481027, "nll": 0.0773771404875677, "ece10": 0.07329989928459148, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.06254867688954732, "nll": 0.17668632212384028, "ece10": 0.140755350841835, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.047071972741837415, "nll": 0.15215558189869144, "ece10": 0.1307969741104173, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 3, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.22975122565804476, "nll": 0.36116660100691395, "ece10": 0.16841056146519137, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7662337662337663, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 4, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.37468326460299095, "nll": 0.6476721302909949, "ece10": 0.16567107872203335, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8333333333333333, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 3, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.28468656453154406, "nll": 0.4541339458033524, "ece10": 0.1509856752434786, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7759259259259258, "confusion_matrix": { "0": { "0": 3, "1": 2, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 17, "accuracy": 0.85, "wilson95": [ 0.6395811352592431, 0.9476312541037835 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.17514543139261463, "nll": 0.36621299124743056, "ece10": 0.19042354386979232, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.8480952380952381, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 0, "other": 1, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 1, "smurfing": 2, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 12, "accuracy": 0.5714285714285714, "wilson95": [ 0.3654655436930671, 0.7553003107462404 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5714285714285714, "brier": 0.6240100193997442, "nll": 1.2666597887491968, "ece10": 0.19151372537484093, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.5529370629370629, "confusion_matrix": { "spelling": { "spelling": 1, "subject_verb_agreement": 0, "word_choice": 2, "punctuation": 1, "none": 0, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 1, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 2, "none": 1, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 3, "none": 2, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.023818581045015164, "nll": 0.10177313900943803, "ece10": 0.09495495401307383, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00848278204341275, "nll": 0.0765706931366441, "ece10": 0.07302839992169119, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 11, "accuracy": 0.7857142857142857, "wilson95": [ 0.524107694133997, 0.9242861328730683 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.3380900952820642, "nll": 0.7164810073127337, "ece10": 0.1910948969296668, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.711111111111111, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.2876151800510466, "nll": 0.5007483727527542, "ece10": 0.14906375859239784, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8097222222222222, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 2, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.31025158120370516, "nll": 0.5807454218921536, "ece10": 0.1965441778906954, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7476190476190476, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 1, "3": 1, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 1, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.4316306187096335, "nll": 0.6130516294750513, "ece10": 0.3109592291360438, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5296442687747036, "confusion_matrix": { "false": { "false": 2, "true": 6, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.019034945003546425, "nll": 0.10696711217281547, "ece10": 0.09920035979582022, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.09008043022263204, "nll": 0.24068249233921488, "ece10": 0.1999615361557588, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.08590209190966835, "nll": 0.20195538954673206, "ece10": 0.17313665999495487, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9403508771929825, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.07001989660600373, "nll": 0.1989376213200186, "ece10": 0.17840834964566973, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.1853298626198104, "nll": 0.3509671514764821, "ece10": 0.14536271783184204, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8771043771043772, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.08953870671589585, "nll": 0.2523665818599001, "ece10": 0.12569556786399644, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571427, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.2649821604501323, "nll": 0.42191213746652756, "ece10": 0.08625819678890052, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8235294117647058, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.09105013576708586, "nll": 0.21535900827833684, "ece10": 0.16168778153859312, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 1, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.47584390355574097, "nll": 1.002511616296356, "ece10": 0.13949282649660577, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7214285714285714, "confusion_matrix": { "0": { "0": 2, "1": 0, "2": 1, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.3022721862187648, "nll": 0.4408067843959623, "ece10": 0.23904125087312728, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6862745098039216, "confusion_matrix": { "false": { "false": 6, "true": 4, "__failed__": 0 }, "true": { "false": 1, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 12, "accuracy": 0.8, "wilson95": [ 0.5481455128483065, 0.9295245065301844 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.274848792852719, "nll": 0.4585150957602677, "ece10": 0.1265336803251292, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.7963800904977376, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.11118193327724277, "nll": 0.24421789656993967, "ece10": 0.14574726707685148, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 1, "tech": 0, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.12657495282452672, "nll": 0.25883321379531693, "ece10": 0.21602908501458562, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 13, "accuracy": 0.6842105263157895, "wilson95": [ 0.46010407675327475, 0.8463562091220773 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6842105263157895, "brier": 0.41350415068227664, "nll": 0.7290567475087811, "ece10": 0.19013653530155125, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.5426136363636364, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.1815748485923362, "nll": 0.33102798815481926, "ece10": 0.16785108188469855, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8583333333333334, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02685858702297512, "nll": 0.10594854118200739, "ece10": 0.09809793048161011, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 13, "accuracy": 0.65, "wilson95": [ 0.43285427668523624, 0.818808175898918 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.65, "brier": 0.35104763367722314, "nll": 0.6138240829869767, "ece10": 0.19434515780477976, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.5409090909090909, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 20, "accuracy": 1.0, "wilson95": [ 0.8388748419471806, 1.0 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.024151665308979484, "nll": 0.107766192754482, "ece10": 0.09780958403197101, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 10, "accuracy": 0.5, "wilson95": [ 0.2992980081982123, 0.7007019918017877 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 10, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.48300374465174567, "nll": 0.6725623012342826, "ece10": 0.20768213038773498, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.5, "confusion_matrix": { "false": { "false": 5, "true": 4, "__failed__": 0 }, "true": { "false": 6, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.17097073255089104, "nll": 0.3796320072628236, "ece10": 0.11651367444951743, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9142857142857143, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 2, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 579, "accuracy": 0.6433333333333333, "wilson95": [ 0.6114896952834068, 0.6739585959873109 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 742, "native_tie_decisions_changed": 9, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6915191053122087, "cluster_bootstrap95": [ 0.597340833046687, 0.6859743696830483 ], "independent_source_groups": 596, "brier": 0.4225864496804264, "nll": 0.7768894796926294, "ece10": 0.06628407391600383, "calibration_n": 1073, "confident_wrong": 16 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 428, "accuracy": 0.7456445993031359, "wilson95": [ 0.7084657124256272, 0.7795574221129166 ], "valid": 573, "failures": 1, "coverage": 0.9982578397212544, "heads": 936, "head_correct": 779, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8322649572649573, "brier": 0.2538670510087708, "nll": 0.517051314015364, "ece10": 0.026423106969369854, "calibration_n": 934, "confident_wrong": 22 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 337, "accuracy": 0.48142857142857143, "wilson95": [ 0.44461666062945593, 0.5184432022499436 ], "valid": 659, "failures": 41, "coverage": 0.9414285714285714, "heads": 1088, "head_correct": 618, "native_tie_decisions_changed": 8, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5680147058823529, "cluster_bootstrap95": [ 0.44266351753938643, 0.5185727589029882 ], "independent_source_groups": 650, "brier": 0.5077813501314857, "nll": 0.9533060705222555, "ece10": 0.05478796998123592, "calibration_n": 1006, "confident_wrong": 11 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 57, "accuracy": 0.1425, "wilson95": [ 0.1116383145426319, 0.18016297590639008 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1228, "native_tie_decisions_changed": 5, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.614, "brier": 0.5162512356905917, "nll": 0.8961348682816132, "ece10": 0.13382541100606432, "calibration_n": 2000, "confident_wrong": 46 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.06414121630543068, "nll": 0.16268448782880016, "ece10": 0.1032041027085602, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 6, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.025374294379355403, "nll": 0.11692204422036491, "ece10": 0.10685736883438705, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "wilson95": [ 0.7735463743160095, 0.9801879358846942 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 26, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.13043873477558005, "nll": 0.25298917093147794, "ece10": 0.15751769045485114, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9270833333333334, "confusion_matrix": { "design": { "design": 5, "inspection": 0, "purchase": 0, "work": 2, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03340054479319391, "nll": 0.13358063974314838, "ece10": 0.11991493130459877, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 37, "accuracy": 0.925, "wilson95": [ 0.8013576647568946, 0.9741639742254119 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.11016708716058601, "nll": 0.2813240643458176, "ece10": 0.16892230617226525, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9198592257415786, "confusion_matrix": { "review": { "review": 5, "it": 0, "documents": 2, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.08265799098807212, "nll": 0.23244520404631622, "ece10": 0.1654877529498151, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9499019607843138, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 7, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 34, "accuracy": 0.85, "wilson95": [ 0.709276756335103, 0.9293881228267964 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.20426976121471987, "nll": 0.42656255308737734, "ece10": 0.14616128203156453, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8144341245270038, "confusion_matrix": { "review": { "review": 2, "it": 1, "documents": 3, "procurement": 0, "maintenance": 2, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 37, "accuracy": 0.925, "wilson95": [ 0.8013576647568946, 0.9741639742254119 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.1444636446404593, "nll": 0.3371874683619661, "ece10": 0.13921657167180362, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9198592257415786, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 2, "it": 0, "review": 5, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 218, "accuracy": 0.545, "wilson95": [ 0.496002466449648, 0.5931414270602654 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 218, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.545, "brier": 0.6044941854611676, "nll": 1.2273945173003797, "ece10": 0.16485176547484798, "calibration_n": 400, "confident_wrong": 2, "macro_f1": 0.4416248213104521, "confusion_matrix": { "sadness": { "sadness": 94, "joy": 7, "love": 8, "anger": 1, "fear": 8, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 27, "joy": 63, "love": 16, "anger": 4, "fear": 4, "surprise": 4, "__failed__": 0 }, "love": { "sadness": 12, "joy": 8, "love": 13, "anger": 1, "fear": 2, "surprise": 1, "__failed__": 0 }, "anger": { "sadness": 27, "joy": 3, "love": 5, "anger": 24, "fear": 3, "surprise": 2, "__failed__": 0 }, "fear": { "sadness": 25, "joy": 1, "love": 0, "anger": 2, "fear": 23, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 5, "joy": 1, "love": 1, "anger": 1, "fear": 1, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 269, "accuracy": 0.8966666666666666, "wilson95": [ 0.8570598275670679, 0.9262434152614588 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 269, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8966666666666666, "brier": 0.145446815541913, "nll": 0.3789443248717196, "ece10": 0.044236348868631536, "calibration_n": 300, "confident_wrong": 3 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 350, "accuracy": 0.875, "wilson95": [ 0.8389809024860414, 0.9038848767632363 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 350, "native_tie_decisions_changed": 1, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.18547093081373622, "nll": 0.30217375023449566, "ece10": 0.034337674769022095, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.8405154540525023, "confusion_matrix": { "false": { "false": 268, "true": 41, "__failed__": 0 }, "true": { "false": 9, "true": 82, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 336, "accuracy": 0.84, "wilson95": [ 0.8008643954521164, 0.8726672444005619 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 336, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.84, "brier": 0.23974763918626718, "nll": 0.4674629613375852, "ece10": 0.04117569709026021, "calibration_n": 400, "confident_wrong": 21, "macro_f1": 0.8256998251934773, "confusion_matrix": { "world": { "world": 86, "sports": 6, "business": 10, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 2, "sports": 121, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 3, "sports": 0, "business": 66, "sci_tech": 3, "__failed__": 0 }, "sci_tech": { "world": 12, "sports": 0, "business": 27, "sci_tech": 63, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 364, "accuracy": 0.91, "wilson95": [ 0.8779172124152713, 0.9342827062306055 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 364, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.91, "brier": 0.13866537438752974, "nll": 0.23438190621807387, "ece10": 0.050637342276648956, "calibration_n": 400, "confident_wrong": 5, "macro_f1": 0.8997326203208555, "confusion_matrix": { "true": { "true": 118, "false": 35, "__failed__": 0 }, "false": { "true": 1, "false": 246, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 227, "accuracy": 0.7566666666666667, "wilson95": [ 0.7050648160221181, 0.8017784587492817 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 227, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7566666666666667, "brier": 0.351066590142623, "nll": 0.9189684754205816, "ece10": 0.07082036674346842, "calibration_n": 300, "confident_wrong": 1 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 369, "accuracy": 0.9225, "wilson95": [ 0.892095045964364, 0.9448670653231555 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 369, "native_tie_decisions_changed": 2, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9225, "brier": 0.107913883747099, "nll": 0.20590903415773457, "ece10": 0.08219450036968114, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.9224956403797714, "confusion_matrix": { "false": { "false": 186, "true": 2, "__failed__": 0 }, "true": { "false": 29, "true": 183, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 262, "accuracy": 0.655, "wilson95": [ 0.6071390167491942, 0.6999121720071738 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 262, "native_tie_decisions_changed": 6, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.655, "brier": 0.39578203795607897, "nll": 0.5648729210527857, "ece10": 0.1279283494005292, "calibration_n": 400, "confident_wrong": 4, "macro_f1": 0.6142557651991614, "confusion_matrix": { "false": { "false": 196, "true": 4, "__failed__": 0 }, "true": { "false": 134, "true": 66, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 72, "accuracy": 0.72, "wilson95": [ 0.6251197129007884, 0.7986031478881379 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 72, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.72, "brier": 0.4407590742810662, "nll": 0.6811361513335981, "ece10": 0.14404752962441147, "calibration_n": 100, "confident_wrong": 13, "macro_f1": 0.6834011759384894, "confusion_matrix": { "yes": { "yes": 53, "no": 1, "__failed__": 0 }, "no": { "yes": 27, "no": 19, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 55, "accuracy": 0.55, "wilson95": [ 0.45244602997442135, 0.6438546202048803 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 55, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.6663572157741074, "nll": 1.613218624049607, "ece10": 0.12453933734988791, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5694057984499161, "confusion_matrix": { "balance": { "balance": 2, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 4, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 7, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 2, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 4, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 4, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 5, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 7, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 4, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 5, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 4, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 144, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.72, "brier": 0.4051103779863701, "nll": 0.6615500013060988, "ece10": 0.14956544452195608, "calibration_n": 200, "confident_wrong": 2 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 42, "accuracy": 0.42, "wilson95": [ 0.3279838267435473, 0.5179351329695703 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 128, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.64, "brier": 0.48079867252239367, "nll": 0.9572356444221386, "ece10": 0.09252162009245764, "calibration_n": 200, "confident_wrong": 4 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 76, "accuracy": 0.76, "wilson95": [ 0.6676766365018553, 0.8330867444305122 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 76, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.3442461744410532, "nll": 0.8894936005903743, "ece10": 0.09391072780407558, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.7280758070618751, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 14, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 4, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 11, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 7, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 2, "syllabus": 1, "news_article": 1, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 4, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 5, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 7, "accuracy": 0.07, "wilson95": [ 0.03431926106727268, 0.137495147390735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 215, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5375, "brier": 0.6997452194688991, "nll": 1.3481203502066967, "ece10": 0.2584461181326303, "calibration_n": 400, "confident_wrong": 43 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 66, "accuracy": 0.66, "wilson95": [ 0.5627772885472462, 0.7453847920265184 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 66, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.66, "brier": 0.5047827939517796, "nll": 1.1861673338050602, "ece10": 0.10537028274173578, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6645062143850675, "confusion_matrix": { "politics": { "politics": 5, "business": 0, "technology": 0, "science": 3, "health": 3, "sports": 0, "entertainment": 1, "world": 1, "culture": 0, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 3, "technology": 7, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 2, "technology": 1, "science": 6, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 6, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 6, "world": 0, "culture": 1, "weather": 0, "__failed__": 0 }, "world": { "politics": 1, "business": 1, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 5, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 1, "world": 0, "culture": 7, "weather": 1, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 2, "culture": 0, "weather": 5, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 42, "accuracy": 0.42, "wilson95": [ 0.3279838267435473, 0.5179351329695703 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 42, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42, "brier": 0.7603943078674277, "nll": 1.820897378001969, "ece10": 0.13618616896555502, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.3932931774108245, "confusion_matrix": { "biology": { "biology": 6, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 8, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 5, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 2, "medicine": 4, "chemistry": 1, "physics": 6, "computer_science": 0, "mathematics": 0, "earth_science": 3, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 1, "medicine": 3, "chemistry": 1, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 4, "engineering": 4, "economics": 1, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 1, "computer_science": 1, "mathematics": 0, "earth_science": 2, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 1, "medicine": 4, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 6, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 2, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 5, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 0, "economics": 4, "other": 0, "__failed__": 0 }, "other": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 0, "earth_science": 3, "engineering": 4, "economics": 1, "other": 1, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 9, "accuracy": 0.09, "wilson95": [ 0.04807254000256516, 0.1622621285271631 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 73, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.365, "brier": 0.5544622517673709, "nll": 1.0683684589486435, "ece10": 0.17689068091224552, "calibration_n": 100, "confident_wrong": 7 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 38, "accuracy": 0.38, "wilson95": [ 0.29097599252478734, 0.47790244704488916 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 129, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.645, "brier": 0.2893461510759236, "nll": 0.5240289987799913, "ece10": 0.09182781609248503, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 86, "accuracy": 0.86, "wilson95": [ 0.7786281178903616, 0.9147365634006086 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.19127676899377172, "nll": 0.37781140437217675, "ece10": 0.11741100967804952, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8616323648103309, "confusion_matrix": { "negative": { "negative": 33, "neutral": 3, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 6, "neutral": 26, "positive": 1, "__failed__": 0 }, "positive": { "negative": 2, "neutral": 2, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 14, "accuracy": 0.14, "wilson95": [ 0.08526343659939134, 0.22137188210963837 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 91, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.455, "brier": 0.2631205123428373, "nll": 0.4014443190989077, "ece10": 0.07827373458186272, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 45, "accuracy": 0.45, "wilson95": [ 0.3561453797951198, 0.5475539700255787 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 235, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7833333333333333, "brier": 0.32440602695588294, "nll": 0.6151710303837069, "ece10": 0.04334530236330467, "calibration_n": 300, "confident_wrong": 7 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 64, "accuracy": 0.64, "wilson95": [ 0.5423540245160874, 0.7272877959859567 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 64, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.64, "brier": 0.5514188898084929, "nll": 1.6123507161976562, "ece10": 0.13236586413071394, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.5997003346793263, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 2, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 3, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 5, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 2, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 3, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 2, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 3, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 3, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 5, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 4, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 2, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 35, "accuracy": 0.35, "wilson95": [ 0.26364248243097527, 0.44745556703112027 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.35, "brier": 0.869326923696637, "nll": 2.0769444612858363, "ece10": 0.2709013210051947, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.33001628001628, "confusion_matrix": { "billing": { "billing": 3, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 5, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 4, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 4, "shipping": 0, "returns": 0, "technical_outage": 1, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 1, "account_access": 0, "shipping": 1, "returns": 2, "technical_outage": 1, "onboarding": 2, "integrations": 1, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 6, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 1, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 1, "pricing": 1, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 3, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 3, "account_access": 0, "shipping": 1, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 2, "compliance": 0, "pricing": 1, "data_privacy": 2, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 9, "accuracy": 0.09, "wilson95": [ 0.04807254000256516, 0.1622621285271631 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 139, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4633333333333333, "brier": 0.7107101171760186, "nll": 1.5302459889319073, "ece10": 0.22078639848682535, "calibration_n": 300, "confident_wrong": 1 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 56, "accuracy": 0.56, "wilson95": [ 0.4622810465167699, 0.6532797336983921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 56, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.56, "brier": 0.6136296259310439, "nll": 1.4139408432434377, "ece10": 0.20033081263920666, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.5292110667110669, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 3, "cancel_trip": 0, "seat": 2, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 1, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 11, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 1, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 3, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 4, "car": 0, "visa_docs": 1, "loyalty": 3, "disruption": 1, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 5, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 4, "car": 2, "visa_docs": 2, "loyalty": 1, "disruption": 1, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.16005058226913804, "nll": 0.3220661018029088, "ece10": 0.10626854495203589, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8675169687777176, "confusion_matrix": { "business": { "business": 25, "science and technology": 0, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 8, "science and technology": 13, "sports": 1, "world": 3, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 74, "accuracy": 0.74, "wilson95": [ 0.6462901055081282, 0.8159530153525186 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 74, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.74, "brier": 0.3673890293031496, "nll": 0.9273363292553882, "ece10": 0.0852307856459126, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.7173500881834215, "confusion_matrix": { "refund not showing up": { "refund not showing up": 1, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 1, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 1, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 2, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 2, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 1, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 1, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 52, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.6658694929010026, "nll": 1.3896594907197342, "ece10": 0.17645835229927737, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.5117375385994931, "confusion_matrix": { "anger": { "anger": 12, "fear": 3, "joy": 0, "love": 0, "sadness": 2, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 1, "fear": 13, "joy": 0, "love": 0, "sadness": 3, "surprise": 0, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 7, "love": 4, "sadness": 3, "surprise": 2, "__failed__": 0 }, "love": { "anger": 1, "fear": 0, "joy": 1, "love": 7, "sadness": 8, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 2, "joy": 2, "love": 3, "sadness": 9, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 2, "fear": 2, "joy": 4, "love": 1, "sadness": 3, "surprise": 4, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2438, "accuracy": 0.8270013568521031, "wilson95": [ 0.8129240920651353, 0.8402275174966687 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2438, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8270013568521031, "brier": 0.2611249999913878, "nll": 0.6663448519643649, "ece10": 0.03686764146168037, "calibration_n": 2948, "confident_wrong": 40, "macro_f1": 0.8157586699771456, "confusion_matrix": { "alarm": { "alarm": 89, "audio": 0, "calendar": 5, "cooking": 0, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 45, "calendar": 3, "cooking": 0, "datetime": 5, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 1, "audio": 0, "calendar": 360, "cooking": 0, "datetime": 9, "email": 0, "general": 3, "iot": 0, "lists": 4, "music": 0, "news": 3, "play": 0, "qa": 7, "recommendation": 10, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 66, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 1, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 88, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 11, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 2, "email": 219, "general": 13, "iot": 0, "lists": 10, "music": 0, "news": 0, "play": 0, "qa": 10, "recommendation": 1, "social": 5, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "general": { "alarm": 1, "audio": 0, "calendar": 10, "cooking": 7, "datetime": 1, "email": 0, "general": 83, "iot": 3, "lists": 0, "music": 1, "news": 1, "play": 3, "qa": 48, "recommendation": 19, "social": 1, "takeaway": 0, "transport": 6, "weather": 3, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 6, "calendar": 0, "cooking": 7, "datetime": 1, "email": 0, "general": 7, "iot": 180, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 13, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 1, "calendar": 4, "cooking": 0, "datetime": 2, "email": 0, "general": 3, "iot": 1, "lists": 123, "music": 5, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 4, "iot": 0, "lists": 1, "music": 64, "news": 1, "play": 3, "qa": 5, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 100, "play": 1, "qa": 20, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 6, "calendar": 1, "cooking": 0, "datetime": 4, "email": 0, "general": 15, "iot": 0, "lists": 1, "music": 47, "news": 1, "play": 299, "qa": 8, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 2, "email": 0, "general": 10, "iot": 1, "lists": 0, "music": 1, "news": 1, "play": 0, "qa": 262, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 2, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 3, "play": 0, "qa": 6, "recommendation": 80, "social": 0, "takeaway": 0, "transport": 3, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 22, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 1, "social": 76, "takeaway": 2, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 2, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 49, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 5, "social": 0, "takeaway": 0, "transport": 116, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 3, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 5, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 1, "weather": 139, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2174, "accuracy": 0.737449118046133, "wilson95": [ 0.7212635393635586, 0.7530166750485398 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2174, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.737449118046133, "brier": 0.37445678636276014, "nll": 0.9728543120876342, "ece10": 0.06367259733008854, "calibration_n": 2948, "confident_wrong": 40, "macro_f1": 0.7291825027531005, "confusion_matrix": { "alarm": { "alarm": 86, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 2, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 37, "calendar": 3, "cooking": 1, "datetime": 4, "email": 0, "general": 11, "iot": 1, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 1, "audio": 0, "calendar": 332, "cooking": 2, "datetime": 11, "email": 3, "general": 10, "iot": 3, "lists": 5, "music": 0, "news": 2, "play": 0, "qa": 7, "recommendation": 17, "social": 1, "takeaway": 4, "transport": 2, "weather": 0, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 65, "datetime": 0, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 88, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 11, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 1, "datetime": 5, "email": 220, "general": 11, "iot": 3, "lists": 5, "music": 0, "news": 1, "play": 0, "qa": 11, "recommendation": 1, "social": 3, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 12, "cooking": 6, "datetime": 8, "email": 1, "general": 70, "iot": 2, "lists": 1, "music": 1, "news": 2, "play": 2, "qa": 50, "recommendation": 16, "social": 1, "takeaway": 0, "transport": 9, "weather": 5, "__failed__": 0 }, "iot": { "alarm": 2, "audio": 8, "calendar": 1, "cooking": 13, "datetime": 2, "email": 0, "general": 6, "iot": 163, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 12, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 1, "datetime": 2, "email": 0, "general": 7, "iot": 2, "lists": 116, "music": 4, "news": 0, "play": 1, "qa": 1, "recommendation": 2, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 3, "email": 0, "general": 3, "iot": 1, "lists": 1, "music": 60, "news": 0, "play": 2, "qa": 7, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 2, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 5, "iot": 0, "lists": 1, "music": 0, "news": 81, "play": 0, "qa": 26, "recommendation": 5, "social": 1, "takeaway": 0, "transport": 1, "weather": 2, "__failed__": 0 }, "play": { "alarm": 0, "audio": 16, "calendar": 1, "cooking": 4, "datetime": 9, "email": 0, "general": 35, "iot": 2, "lists": 6, "music": 99, "news": 0, "play": 181, "qa": 19, "recommendation": 6, "social": 0, "takeaway": 1, "transport": 6, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 2, "calendar": 1, "cooking": 0, "datetime": 10, "email": 0, "general": 13, "iot": 1, "lists": 0, "music": 1, "news": 2, "play": 2, "qa": 246, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 3, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 1, "qa": 6, "recommendation": 80, "social": 0, "takeaway": 1, "transport": 2, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 1, "email": 2, "general": 21, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 3, "social": 64, "takeaway": 6, "transport": 2, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 3, "datetime": 1, "email": 0, "general": 2, "iot": 2, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 45, "transport": 1, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 2, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 3, "social": 0, "takeaway": 1, "transport": 114, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 2, "email": 0, "general": 4, "iot": 2, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 6, "recommendation": 7, "social": 0, "takeaway": 0, "transport": 2, "weather": 126, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 806, "accuracy": 0.6871270247229326, "wilson95": [ 0.6600185700935209, 0.7130138351719421 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 806, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6871270247229326, "cluster_bootstrap95": [ 0.6543110021440749, 0.7183272883144289 ], "independent_source_groups": 47, "brier": 0.42826213359034, "nll": 0.7342980645247908, "ece10": 0.0405037409372302, "calibration_n": 1173, "confident_wrong": 1, "macro_f1": 0.6620555759545379, "confusion_matrix": { "negative": { "negative": 118, "neutral": 42, "positive": 22, "__failed__": 0 }, "neutral": { "negative": 78, "neutral": 401, "positive": 111, "__failed__": 0 }, "positive": { "negative": 43, "neutral": 71, "positive": 287, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "tev1": { "label": "Tev1 4B", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 175, "accuracy": 0.875, "wilson95": [ 0.8219857499741765, 0.9138802552428735 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 175, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.16138167616401936, "nll": 0.2927945320102345, "ece10": 0.03838515347067242, "calibration_n": 200, "confident_wrong": 4, "macro_f1": 0.878800047528896, "confusion_matrix": { "world": { "world": 44, "sports": 0, "business": 2, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 41, "sci_tech": 8, "__failed__": 0 }, "sci_tech": { "world": 1, "sports": 0, "business": 7, "sci_tech": 44, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 118, "accuracy": 0.59, "wilson95": [ 0.5207645903369208, 0.6558432509151711 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 118, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.59, "brier": 0.6529938268976029, "nll": 1.5301414289892774, "ece10": 0.20884021654990192, "calibration_n": 200, "confident_wrong": 26, "macro_f1": 0.49005405250235023, "confusion_matrix": { "sadness": { "sadness": 43, "joy": 3, "love": 2, "anger": 5, "fear": 3, "surprise": 1, "__failed__": 0 }, "joy": { "sadness": 18, "joy": 44, "love": 10, "anger": 4, "fear": 3, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 2, "joy": 4, "love": 5, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 }, "anger": { "sadness": 7, "joy": 4, "love": 0, "anger": 14, "fear": 1, "surprise": 2, "__failed__": 0 }, "fear": { "sadness": 4, "joy": 2, "love": 1, "anger": 1, "fear": 11, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 190, "accuracy": 0.95, "wilson95": [ 0.9104218518612239, 0.972617354399236 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 190, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.1267080637319555, "nll": 0.2467273363055666, "ece10": 0.12891191270582358, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.9493875898370281, "confusion_matrix": { "false": { "false": 106, "true": 3, "__failed__": 0 }, "true": { "false": 7, "true": 84, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 175, "accuracy": 0.875, "wilson95": [ 0.8219857499741765, 0.9138802552428735 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 175, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.20770220420432448, "nll": 0.33893879713801356, "ece10": 0.04814043914350222, "calibration_n": 200, "confident_wrong": 5, "macro_f1": 0.8569753139391859, "confusion_matrix": { "true": { "true": 52, "false": 25, "__failed__": 0 }, "false": { "true": 0, "false": 123, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 8, "accuracy": 0.6666666666666666, "wilson95": [ 0.39062208887279953, 0.8618799089087867 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.3239829844435575, "nll": 0.5017303215348489, "ece10": 0.16881287340594747, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.6933333333333334, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 1, "documents": 0, "it": 0, "review": 1, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.33443111301280215, "nll": 0.5797862836235411, "ece10": 0.17224532783406096, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8533333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 1, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0007326302410192647, "nll": 0.014254309271271214, "ece10": 0.014055496359896705, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.013328641080910537, "nll": 0.0394676060648158, "ece10": 0.03529649386311504, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 96, "accuracy": 0.96, "wilson95": [ 0.9016292856411208, 0.9843366960084523 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 96, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.96, "brier": 0.07563144258905968, "nll": 0.18157610706035146, "ece10": 0.049531700978388334, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.9597919167667067, "confusion_matrix": { "label_000": { "label_000": 23, "label_001": 2, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 23, "label_002": 1, "label_003": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.7389182599470188, "nll": 1.833062557056154, "ece10": 0.2290929841254875, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.4692623057840448, "confusion_matrix": { "label_000": { "label_000": 11, "label_001": 2, "label_002": 1, "label_003": 0, "label_004": 3, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 2, "label_001": 11, "label_002": 1, "label_003": 0, "label_004": 2, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 9, "label_003": 1, "label_004": 4, "label_005": 2, "__failed__": 0 }, "label_003": { "label_000": 3, "label_001": 0, "label_002": 3, "label_003": 4, "label_004": 7, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 1, "label_001": 2, "label_002": 2, "label_003": 2, "label_004": 9, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 2, "label_001": 1, "label_002": 4, "label_003": 1, "label_004": 4, "label_005": 4, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "label_000": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_001": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 1.2974547140299377e-05, "nll": 0.0019734293465983282, "ece10": 0.001969720683912768, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 57, "accuracy": 0.5135135135135135, "wilson95": [ 0.4216469238103697, 0.6044760463330521 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 57, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5135135135135135, "brier": 0.7034823091663751, "nll": 1.4079303285688194, "ece10": 0.2906719318037865, "calibration_n": 111, "confident_wrong": 17 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 69, "accuracy": 0.9583333333333334, "wilson95": [ 0.8845072325868389, 0.9857291925330748 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9583333333333334, "cluster_bootstrap95": [ 0.8888888888888888, 1.0 ], "independent_source_groups": 36, "brier": 0.05587089971175063, "nll": 0.10785224997316065, "ece10": 0.03435316080231132, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 17, "accuracy": 0.8947368421052632, "wilson95": [ 0.6860591728889099, 0.9706414397011299 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8947368421052632, "brier": 0.16410229342458776, "nll": 0.2820699379166834, "ece10": 0.1279971756017147, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8944444444444444, "confusion_matrix": { "false": { "false": 9, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02439252599967174, "nll": 0.09624388287855125, "ece10": 0.08937207363542223, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.23959143143187359, "nll": 0.4007269999837556, "ece10": 0.11561720708373918, "calibration_n": 22, "confident_wrong": 1, "macro_f1": 0.8562091503267973, "confusion_matrix": { "false": { "false": 7, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.015909721674314953, "nll": 0.04057946557339949, "ece10": 0.03535254190856464, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.13130716539521858, "nll": 0.2700663325508315, "ece10": 0.19025018325700807, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.013794068878994327, "nll": 0.034447241029231315, "ece10": 0.02995269090885705, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.2621391502909917, "nll": 0.4531810286663024, "ece10": 0.15392294439533893, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.7666666666666666, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 4, "refactor": 0, "nit": 0, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.10343735191420332, "nll": 0.18801326835171103, "ece10": 0.06460125503828805, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9111111111111111, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 2, "test": 0, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.15352932959353244, "nll": 0.2551675789229588, "ece10": 0.0945907681602278, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8785714285714286, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.1834656802976984, "nll": 0.30843685512909047, "ece10": 0.15722132015439763, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.2548060973200685, "nll": 0.5294613645679067, "ece10": 0.1063992251806621, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.877948717948718, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 1, "threat": 1, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 6, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 19, "accuracy": 0.8636363636363636, "wilson95": [ 0.6666501283342828, 0.9525099628594474 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8636363636363636, "brier": 0.21923779565708432, "nll": 0.37857376849153, "ece10": 0.09996051910868513, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8565656565656565, "confusion_matrix": { "news_report": { "news_report": 4, "opinion": 0, "press_release": 1, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 2, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 2, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0022913491721449483, "nll": 0.013315733605914265, "ece10": 0.012682330246467966, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0010986152273239369, "nll": 0.010394483421931375, "ece10": 0.010092338683601534, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.05776500934737494, "nll": 0.13872312990845223, "ece10": 0.12050604216394817, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0004987172355024221, "nll": 0.008274548802106112, "ece10": 0.008094692911717057, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.1308527269367834, "nll": 0.23530056479301167, "ece10": 0.08207026160452244, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.9314285714285713, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0385939123028444, "nll": 0.06942064677662879, "ece10": 0.055292896440211965, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 3, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.1379384728692875, "nll": 0.25071481444879695, "ece10": 0.11000256523996418, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8888888888888888, "confusion_matrix": { "false": { "false": 8, "true": 1, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.4942620096873859, "nll": 0.7026856963881625, "ece10": 0.21993280127449863, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5551948051948052, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.32798809656795375, "nll": 0.47805792601515074, "ece10": 0.21198263074782075, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7027568922305765, "confusion_matrix": { "0": { "0": 2, "1": 3, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.1198056054663085, "nll": 0.2586141996958866, "ece10": 0.15908114623690045, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 4, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 3, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 12, "accuracy": 0.5714285714285714, "wilson95": [ 0.3654655436930671, 0.7553003107462404 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5714285714285714, "brier": 0.7135292187811245, "nll": 1.3544131134720376, "ece10": 0.31293583141307474, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.5844444444444445, "confusion_matrix": { "spelling": { "spelling": 2, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 1, "none": 1, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 0, "punctuation": 0, "none": 1, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 1, "punctuation": 0, "none": 3, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 2, "none": 3, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.05189542861567264, "nll": 0.12063088085729662, "ece10": 0.10317979385669873, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.005017688407687723, "nll": 0.019584990332252944, "ece10": 0.017919270593419384, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 12, "accuracy": 0.8571428571428571, "wilson95": [ 0.6005862050302965, 0.9599060787285351 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8571428571428571, "brier": 0.2323077278410345, "nll": 0.4927303373594958, "ece10": 0.10668940110697595, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.75, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.2834901181042083, "nll": 0.44521988932566214, "ece10": 0.21005540534299474, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8875, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.27446699037724426, "nll": 0.5233384733490999, "ece10": 0.22879551868067802, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.8031746031746032, "confusion_matrix": { "0": { "0": 2, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.30694987930606477, "nll": 0.47221976710625585, "ece10": 0.11734351411575974, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8131868131868132, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.019507296108699255, "nll": 0.06823837207747799, "ece10": 0.06162365733210588, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04000846628310268, "nll": 0.09835484451596414, "ece10": 0.08447071877593108, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.1580705085715338, "nll": 0.2990778949141362, "ece10": 0.24473833467585573, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03567890246199531, "nll": 0.11923321506950596, "ece10": 0.10853431334228575, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.12543239269744372, "nll": 0.25799925684058705, "ece10": 0.20987067993879058, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.08782494643710871, "nll": 0.19278457537061738, "ece10": 0.01931747682609333, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571427, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 1, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.3618580425478494, "nll": 0.5096587932598172, "ece10": 0.13016365221417567, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6136363636363636, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.040681281783961465, "nll": 0.09399471859289588, "ece10": 0.0806821953176735, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 4, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.40944522081685597, "nll": 1.1600662264107906, "ece10": 0.21294541903433745, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.6547619047619048, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 2, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 10, "accuracy": 0.625, "wilson95": [ 0.38641040545502714, 0.8151876744113636 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.5116285424419473, "nll": 0.7543175168555418, "ece10": 0.24017532025712196, "calibration_n": 16, "confident_wrong": 1, "macro_f1": 0.625, "confusion_matrix": { "false": { "false": 5, "true": 5, "__failed__": 0 }, "true": { "false": 1, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 9, "accuracy": 0.6, "wilson95": [ 0.357468301206246, 0.8017550385865841 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.3485136609538647, "nll": 0.49049723224923636, "ece10": 0.2489999566754448, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.55, "confusion_matrix": { "false": { "false": 2, "true": 6, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.1232435887000021, "nll": 0.19746965857945945, "ece10": 0.14231197317405, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7918181818181818, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 1, "tech": 0, "sales": 3, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 1, "account": 0, "other": 1, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.1967927514362924, "nll": 0.32083739260959065, "ece10": 0.17349594653262418, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9403508771929825, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 13, "accuracy": 0.6842105263157895, "wilson95": [ 0.46010407675327475, 0.8463562091220773 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6842105263157895, "brier": 0.4423028828787069, "nll": 0.7428192092912693, "ece10": 0.17776395522126276, "calibration_n": 19, "confident_wrong": 1, "macro_f1": 0.5446428571428571, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 2, "2": 0, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.3043432935967904, "nll": 0.46906524849099446, "ece10": 0.12619387259287768, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7571428571428571, "confusion_matrix": { "false": { "false": 8, "true": 3, "__failed__": 0 }, "true": { "false": 1, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.035586767519867814, "nll": 0.0922194744060782, "ece10": 0.07998940050247974, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 16, "accuracy": 0.8, "wilson95": [ 0.5839825677481064, 0.919342337420202 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.36816918083232786, "nll": 0.5956831673562694, "ece10": 0.21367313543458583, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.780952380952381, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.05724110656832345, "nll": 0.07825939567801563, "ece10": 0.04526332589855993, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 14, "accuracy": 0.7, "wilson95": [ 0.4810271816464765, 0.8545227551323956 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.3639455242114481, "nll": 0.5915779035022718, "ece10": 0.12686965995138247, "calibration_n": 20, "confident_wrong": 1, "macro_f1": 0.6875, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.12915527719265915, "nll": 0.25772894761903364, "ece10": 0.09016598255296783, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9142857142857143, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 2, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 584, "accuracy": 0.6488888888888888, "wilson95": [ 0.6171318176276369, 0.6793803609016357 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 750, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6989748369058714, "cluster_bootstrap95": [ 0.6084470125199947, 0.6957499313686875 ], "independent_source_groups": 596, "brier": 0.4071677999696497, "nll": 0.8586730782591033, "ece10": 0.08088661257490634, "calibration_n": 1073, "confident_wrong": 54 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 432, "accuracy": 0.7526132404181185, "wilson95": [ 0.7157120359592749, 0.7861557265094404 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 780, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.25351424004391776, "nll": 0.5170764700854521, "ece10": 0.06925098376903749, "calibration_n": 936, "confident_wrong": 40 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 316, "accuracy": 0.4514285714285714, "wilson95": [ 0.41492879364937185, 0.4884585400352113 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 616, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5661764705882353, "cluster_bootstrap95": [ 0.4145212765957447, 0.4865286025484169 ], "independent_source_groups": 650, "brier": 0.596881832494507, "nll": 1.264049270297487, "ece10": 0.18177056817999138, "calibration_n": 1088, "confident_wrong": 89 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 60, "accuracy": 0.15, "wilson95": [ 0.11834499530337544, 0.18831361073063213 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1321, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6605, "brier": 0.45702468182056055, "nll": 0.7880949667248284, "ece10": 0.05393505015826044, "calibration_n": 2000, "confident_wrong": 36 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.020813911392673506, "nll": 0.03858978463330438, "ece10": 0.030186074040397165, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025642, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 6, "purchase": 0, "work": 1, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.028110411149276954, "nll": 0.04442127701990744, "ece10": 0.03150544695766376, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 0, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.019981298794358886, "nll": 0.060305945009876516, "ece10": 0.05345395617073601, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.01490664883116259, "nll": 0.05129296851984894, "ece10": 0.04623977805146834, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 39, "accuracy": 0.975, "wilson95": [ 0.8711863103652591, 0.9955731684973186 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.975, "brier": 0.03986260719688995, "nll": 0.11434831784205361, "ece10": 0.09598705621923637, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9749019607843138, "confusion_matrix": { "review": { "review": 8, "it": 0, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 7, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 39, "accuracy": 0.975, "wilson95": [ 0.8711863103652591, 0.9955731684973186 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 39, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.975, "brier": 0.057464710879844484, "nll": 0.14507104191166403, "ece10": 0.09025181037129987, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9749019607843138, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 7, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 8, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 37, "accuracy": 0.925, "wilson95": [ 0.8013576647568946, 0.9741639742254119 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.13208165415874232, "nll": 0.26149574549548954, "ece10": 0.0765538143934733, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9185520361990951, "confusion_matrix": { "review": { "review": 5, "it": 0, "documents": 1, "procurement": 1, "maintenance": 1, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 37, "accuracy": 0.925, "wilson95": [ 0.8013576647568946, 0.9741639742254119 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 37, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.925, "brier": 0.11174524084569734, "nll": 0.23745071254227565, "ece10": 0.115036326653454, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9228991596638656, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 7, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 1, "documents": 1, "it": 0, "review": 6, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 231, "accuracy": 0.5775, "wilson95": [ 0.5285810181257037, 0.6249445762524805 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 231, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5775, "brier": 0.5960228114335127, "nll": 1.4001097516678076, "ece10": 0.1822623166239545, "calibration_n": 400, "confident_wrong": 21, "macro_f1": 0.4861265833353212, "confusion_matrix": { "sadness": { "sadness": 95, "joy": 9, "love": 7, "anger": 3, "fear": 5, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 28, "joy": 66, "love": 16, "anger": 3, "fear": 3, "surprise": 2, "__failed__": 0 }, "love": { "sadness": 11, "joy": 7, "love": 13, "anger": 2, "fear": 2, "surprise": 2, "__failed__": 0 }, "anger": { "sadness": 21, "joy": 3, "love": 6, "anger": 31, "fear": 1, "surprise": 2, "__failed__": 0 }, "fear": { "sadness": 18, "joy": 2, "love": 0, "anger": 6, "fear": 24, "surprise": 2, "__failed__": 0 }, "surprise": { "sadness": 4, "joy": 1, "love": 1, "anger": 1, "fear": 1, "surprise": 2, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 268, "accuracy": 0.8933333333333333, "wilson95": [ 0.8532967116826325, 0.9234241509540578 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 268, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8933333333333333, "brier": 0.1376868534927335, "nll": 0.3559518493807075, "ece10": 0.023509439132861347, "calibration_n": 300, "confident_wrong": 3 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 333, "accuracy": 0.8325, "wilson95": [ 0.7927798983351753, 0.8658944259325176 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 333, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8325, "brier": 0.24340082095199667, "nll": 0.3837170092630204, "ece10": 0.05296315987673038, "calibration_n": 400, "confident_wrong": 3, "macro_f1": 0.7869210892466707, "confusion_matrix": { "false": { "false": 259, "true": 50, "__failed__": 0 }, "true": { "false": 17, "true": 74, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 352, "accuracy": 0.88, "wilson95": [ 0.8444860470597602, 0.9082846092461745 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 352, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.1808856715541557, "nll": 0.36221468918469574, "ece10": 0.04503413506330149, "calibration_n": 400, "confident_wrong": 18, "macro_f1": 0.8698979756153669, "confusion_matrix": { "world": { "world": 87, "sports": 6, "business": 9, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 2, "sports": 121, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 64, "sci_tech": 6, "__failed__": 0 }, "sci_tech": { "world": 6, "sports": 0, "business": 16, "sci_tech": 80, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 339, "accuracy": 0.8475, "wilson95": [ 0.8089761434535727, 0.8794128119840912 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 339, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8475, "brier": 0.21362574188526218, "nll": 0.33925987824526366, "ece10": 0.04790462845392525, "calibration_n": 400, "confident_wrong": 10, "macro_f1": 0.8221561381642724, "confusion_matrix": { "true": { "true": 94, "false": 59, "__failed__": 0 }, "false": { "true": 2, "false": 245, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 228, "accuracy": 0.76, "wilson95": [ 0.7085788119154636, 0.8048468430477725 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 228, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.3914796484218233, "nll": 1.0422605489202421, "ece10": 0.08475589527598458, "calibration_n": 300, "confident_wrong": 18 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 380, "accuracy": 0.95, "wilson95": [ 0.9240363649362803, 0.9674025701628527 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 380, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.11740109864032677, "nll": 0.23339892136315185, "ece10": 0.13128613351660692, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.9499198717948718, "confusion_matrix": { "false": { "false": 182, "true": 6, "__failed__": 0 }, "true": { "false": 14, "true": 198, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 277, "accuracy": 0.6925, "wilson95": [ 0.6456251644125997, 0.7357126022686962 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 277, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6925, "brier": 0.36443422574367285, "nll": 0.5413808382837016, "ece10": 0.06364906584875757, "calibration_n": 400, "confident_wrong": 10, "macro_f1": 0.6749278167967177, "confusion_matrix": { "false": { "false": 185, "true": 15, "__failed__": 0 }, "true": { "false": 108, "true": 92, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 77, "accuracy": 0.77, "wilson95": [ 0.6784561697712622, 0.8415673411969654 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 77, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.77, "brier": 0.3302987684112278, "nll": 0.5137905047265058, "ece10": 0.07964565687390582, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.7519145723222953, "confusion_matrix": { "yes": { "yes": 52, "no": 2, "__failed__": 0 }, "no": { "yes": 21, "no": 25, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 57, "accuracy": 0.57, "wilson95": [ 0.47215389549212305, 0.6626670147588988 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 57, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.57, "brier": 0.7310951971311787, "nll": 2.2007242818483728, "ece10": 0.27261832947757664, "calibration_n": 100, "confident_wrong": 13, "macro_f1": 0.585954534595839, "confusion_matrix": { "balance": { "balance": 3, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 4, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 8, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 2, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 4, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 3, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 5, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 5, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 5, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 5, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 4, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 4, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 2, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 143, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.715, "brier": 0.37426716360813983, "nll": 0.5838485663042647, "ece10": 0.13208317303539135, "calibration_n": 200, "confident_wrong": 2 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 125, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.5259197046256207, "nll": 1.1328766578620033, "ece10": 0.123898024877335, "calibration_n": 200, "confident_wrong": 12 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 77, "accuracy": 0.77, "wilson95": [ 0.6784561697712622, 0.8415673411969654 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 77, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.77, "brier": 0.3568116458488143, "nll": 1.1109215666536132, "ece10": 0.1434724335860801, "calibration_n": 100, "confident_wrong": 7, "macro_f1": 0.7104991685874039, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 14, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 3, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 11, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 1, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 6, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 2, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 6, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 4, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 216, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.54, "brier": 0.7237022661200438, "nll": 1.6653448518692893, "ece10": 0.29595827741567016, "calibration_n": 400, "confident_wrong": 60 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 64, "accuracy": 0.64, "wilson95": [ 0.5423540245160874, 0.7272877959859567 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 64, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.64, "brier": 0.5606317114772609, "nll": 1.6254816388775686, "ece10": 0.16767944343124902, "calibration_n": 100, "confident_wrong": 10, "macro_f1": 0.6356887770782296, "confusion_matrix": { "politics": { "politics": 2, "business": 1, "technology": 1, "science": 4, "health": 3, "sports": 0, "entertainment": 1, "world": 1, "culture": 0, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 1, "technology": 9, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 2, "technology": 1, "science": 6, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 6, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 7, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "world": { "politics": 0, "business": 4, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 3, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 3, "world": 0, "culture": 6, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 6, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 40, "accuracy": 0.4, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.8680690167030625, "nll": 2.3379153750078476, "ece10": 0.2867270553233339, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.36451452056715217, "confusion_matrix": { "biology": { "biology": 7, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 8, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 5, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 3, "medicine": 3, "chemistry": 1, "physics": 4, "computer_science": 0, "mathematics": 0, "earth_science": 3, "engineering": 1, "economics": 1, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 2, "chemistry": 2, "physics": 0, "computer_science": 3, "mathematics": 0, "earth_science": 5, "engineering": 1, "economics": 1, "other": 1, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 1, "chemistry": 1, "physics": 1, "computer_science": 1, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 1, "medicine": 4, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 6, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 1, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 0, "economics": 3, "other": 1, "__failed__": 0 }, "other": { "biology": 2, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 1, "mathematics": 0, "earth_science": 4, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 9, "accuracy": 0.09, "wilson95": [ 0.04807254000256516, 0.1622621285271631 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 77, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.385, "brier": 0.5608064268875406, "nll": 1.2995826601377862, "ece10": 0.20038984213717706, "calibration_n": 100, "confident_wrong": 15 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 36, "accuracy": 0.36, "wilson95": [ 0.2727122040140433, 0.4576459754839126 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 126, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.63, "brier": 0.2751645752411914, "nll": 0.5052232092653705, "ece10": 0.07171129386502993, "calibration_n": 100, "confident_wrong": 1 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 87, "accuracy": 0.87, "wilson95": [ 0.7901964855992357, 0.9224283257275948 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 87, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.18134939073696768, "nll": 0.4078841142086368, "ece10": 0.06828975648034305, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.8721742166361176, "confusion_matrix": { "negative": { "negative": 33, "neutral": 3, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 5, "neutral": 28, "positive": 0, "__failed__": 0 }, "positive": { "negative": 2, "neutral": 3, "positive": 26, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 11, "accuracy": 0.11, "wilson95": [ 0.06254196357064568, 0.18631296503080313 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 89, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.445, "brier": 0.2537039965365266, "nll": 0.39699698168246056, "ece10": 0.09629713732142399, "calibration_n": 100, "confident_wrong": 2 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 230, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7666666666666667, "brier": 0.3037425475963935, "nll": 0.714667999067625, "ece10": 0.08019799546104647, "calibration_n": 300, "confident_wrong": 10 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "order_status": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "refund_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 6 }, "shipping_delay": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "damaged_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "pricing_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "speak_to_human": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 7 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 4 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 9 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 7 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 1 }, "promo_code": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 2 }, "partner_inquiry": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 5 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 3 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 42, "accuracy": 0.42, "wilson95": [ 0.3279838267435473, 0.5179351329695703 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 42, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42, "brier": 0.894083093253601, "nll": 2.5284280143597426, "ece10": 0.362884705596495, "calibration_n": 100, "confident_wrong": 13, "macro_f1": 0.4165340406719717, "confusion_matrix": { "billing": { "billing": 3, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 4, "shipping": 1, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 6, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 3, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 1, "onboarding": 2, "integrations": 3, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 6, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 3, "pricing": 0, "data_privacy": 1, "other": 2, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 5, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 1, "account_access": 0, "shipping": 3, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 1, "data_privacy": 1, "other": 1, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 9, "accuracy": 0.09, "wilson95": [ 0.04807254000256516, 0.1622621285271631 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 137, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45666666666666667, "brier": 0.7241160329444578, "nll": 1.7277471462939877, "ece10": 0.26795910644722465, "calibration_n": 300, "confident_wrong": 15 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 59, "accuracy": 0.59, "wilson95": [ 0.4920143005341078, 0.6813268697886348 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 59, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.59, "brier": 0.6848016665401839, "nll": 1.8530927185803296, "ece10": 0.2994832981343656, "calibration_n": 100, "confident_wrong": 21, "macro_f1": 0.5579180761714302, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 4, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 1, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 2, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 11, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 2, "loyalty": 2, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 4, "car": 0, "visa_docs": 0, "loyalty": 5, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 6, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 2, "refund": 0, "hotel": 4, "car": 1, "visa_docs": 1, "loyalty": 1, "disruption": 1, "other": 1, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 94, "accuracy": 0.94, "wilson95": [ 0.8752318455410406, 0.9722138760368118 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 94, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.94, "brier": 0.08306237367746155, "nll": 0.19149410808368622, "ece10": 0.05551473724461374, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.9393627450980392, "confusion_matrix": { "business": { "business": 22, "science and technology": 3, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 1, "science and technology": 22, "sports": 1, "world": 1, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "refund not showing up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "activating a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 50, "accuracy": 0.5, "wilson95": [ 0.4038315303659956, 0.5961684696340044 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 50, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.7190075727052359, "nll": 1.856208911669573, "ece10": 0.22407755693684467, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.485838779956427, "confusion_matrix": { "anger": { "anger": 11, "fear": 3, "joy": 1, "love": 0, "sadness": 2, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 1, "fear": 13, "joy": 1, "love": 0, "sadness": 2, "surprise": 0, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 8, "love": 2, "sadness": 4, "surprise": 2, "__failed__": 0 }, "love": { "anger": 2, "fear": 1, "joy": 1, "love": 7, "sadness": 6, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 2, "fear": 3, "joy": 2, "love": 1, "sadness": 8, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 2, "fear": 2, "joy": 4, "love": 1, "sadness": 4, "surprise": 3, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2421, "accuracy": 0.8212347354138398, "wilson95": [ 0.8069881793407908, 0.8346451964424336 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2421, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8212347354138398, "brier": 0.2663047436237839, "nll": 0.6825759821703505, "ece10": 0.06797853788036225, "calibration_n": 2948, "confident_wrong": 123, "macro_f1": 0.8206480675292694, "confusion_matrix": { "alarm": { "alarm": 93, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 46, "calendar": 3, "cooking": 0, "datetime": 2, "email": 0, "general": 6, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 3, "audio": 0, "calendar": 360, "cooking": 0, "datetime": 4, "email": 0, "general": 6, "iot": 0, "lists": 1, "music": 0, "news": 3, "play": 0, "qa": 3, "recommendation": 18, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 68, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 90, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 7, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 2, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 1, "calendar": 5, "cooking": 0, "datetime": 0, "email": 221, "general": 26, "iot": 1, "lists": 8, "music": 0, "news": 0, "play": 0, "qa": 5, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 8, "cooking": 7, "datetime": 0, "email": 0, "general": 105, "iot": 2, "lists": 0, "music": 1, "news": 1, "play": 3, "qa": 28, "recommendation": 24, "social": 1, "takeaway": 1, "transport": 3, "weather": 3, "__failed__": 0 }, "iot": { "alarm": 3, "audio": 3, "calendar": 0, "cooking": 8, "datetime": 1, "email": 0, "general": 11, "iot": 175, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 13, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 0, "datetime": 0, "email": 0, "general": 9, "iot": 1, "lists": 119, "music": 5, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 9, "iot": 0, "lists": 0, "music": 69, "news": 0, "play": 1, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 0, "lists": 0, "music": 0, "news": 97, "play": 1, "qa": 18, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 1, "__failed__": 0 }, "play": { "alarm": 0, "audio": 2, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 20, "iot": 1, "lists": 2, "music": 95, "news": 1, "play": 261, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 22, "iot": 0, "lists": 0, "music": 1, "news": 1, "play": 1, "qa": 250, "recommendation": 5, "social": 0, "takeaway": 0, "transport": 3, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 3, "play": 1, "qa": 3, "recommendation": 83, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 24, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 73, "takeaway": 3, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 1, "datetime": 1, "email": 0, "general": 1, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 51, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 3, "social": 0, "takeaway": 0, "transport": 117, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 143, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2143, "accuracy": 0.7269335142469471, "wilson95": [ 0.7105629866559836, 0.7427133896892931 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2143, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7269335142469471, "brier": 0.4010135665012642, "nll": 1.0573288074849039, "ece10": 0.11847349571719515, "calibration_n": 2948, "confident_wrong": 151, "macro_f1": 0.7372468834305076, "confusion_matrix": { "alarm": { "alarm": 89, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 2, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 2, "audio": 41, "calendar": 1, "cooking": 0, "datetime": 3, "email": 0, "general": 11, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 3, "audio": 0, "calendar": 340, "cooking": 0, "datetime": 6, "email": 1, "general": 18, "iot": 1, "lists": 2, "music": 0, "news": 3, "play": 1, "qa": 2, "recommendation": 19, "social": 1, "takeaway": 1, "transport": 2, "weather": 0, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 65, "datetime": 0, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 1, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 94, "email": 0, "general": 1, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 0, "datetime": 2, "email": 215, "general": 32, "iot": 1, "lists": 8, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 10, "cooking": 8, "datetime": 1, "email": 1, "general": 101, "iot": 2, "lists": 1, "music": 1, "news": 2, "play": 3, "qa": 25, "recommendation": 21, "social": 0, "takeaway": 0, "transport": 7, "weather": 4, "__failed__": 0 }, "iot": { "alarm": 3, "audio": 6, "calendar": 1, "cooking": 10, "datetime": 2, "email": 0, "general": 17, "iot": 159, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 10, "transport": 1, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 0, "datetime": 0, "email": 0, "general": 14, "iot": 2, "lists": 112, "music": 4, "news": 0, "play": 1, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 0, "lists": 0, "music": 67, "news": 0, "play": 1, "qa": 2, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 0, "email": 1, "general": 18, "iot": 0, "lists": 1, "music": 0, "news": 73, "play": 0, "qa": 19, "recommendation": 4, "social": 0, "takeaway": 1, "transport": 1, "weather": 2, "__failed__": 0 }, "play": { "alarm": 0, "audio": 12, "calendar": 2, "cooking": 2, "datetime": 6, "email": 0, "general": 56, "iot": 2, "lists": 5, "music": 144, "news": 0, "play": 141, "qa": 6, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 4, "weather": 1, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 2, "calendar": 4, "cooking": 0, "datetime": 3, "email": 0, "general": 36, "iot": 0, "lists": 1, "music": 1, "news": 1, "play": 0, "qa": 224, "recommendation": 9, "social": 0, "takeaway": 0, "transport": 3, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 1, "email": 0, "general": 4, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 1, "qa": 0, "recommendation": 82, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 1, "general": 35, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 59, "takeaway": 4, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 2, "datetime": 0, "email": 0, "general": 6, "iot": 2, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 42, "transport": 1, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 1, "datetime": 0, "email": 0, "general": 2, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 5, "social": 0, "takeaway": 0, "transport": 112, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 0, "datetime": 3, "email": 0, "general": 10, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 1, "weather": 127, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 857, "accuracy": 0.7306052855924978, "wilson95": [ 0.704494454705101, 0.7552106278657772 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 857, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7306052855924978, "cluster_bootstrap95": [ 0.6992235240752469, 0.7618598165306781 ], "independent_source_groups": 47, "brier": 0.38367808024441763, "nll": 0.6746339461611371, "ece10": 0.04382343840962922, "calibration_n": 1173, "confident_wrong": 12, "macro_f1": 0.7031692134970952, "confusion_matrix": { "negative": { "negative": 123, "neutral": 46, "positive": 13, "__failed__": 0 }, "neutral": { "negative": 79, "neutral": 442, "positive": 69, "__failed__": 0 }, "positive": { "negative": 33, "neutral": 76, "positive": 292, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "von": { "label": "Von", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 165, "accuracy": 0.825, "wilson95": [ 0.7663556885175099, 0.8713948493372666 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 165, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.825, "brier": 0.29286658477111055, "nll": 0.5780574952378827, "ece10": 0.11922036941282368, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.8230702741407528, "confusion_matrix": { "world": { "world": 42, "sports": 1, "business": 0, "sci_tech": 7, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 0, "sci_tech": 1, "__failed__": 0 }, "business": { "world": 6, "sports": 0, "business": 27, "sci_tech": 18, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 0, "sci_tech": 50, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 177, "accuracy": 0.885, "wilson95": [ 0.8333528310151243, 0.9221362676743804 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 177, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.885, "brier": 0.21465025121702994, "nll": 0.4651887884903784, "ece10": 0.1403726649845746, "calibration_n": 200, "confident_wrong": 1, "macro_f1": 0.710241707410603, "confusion_matrix": { "sadness": { "sadness": 53, "joy": 0, "love": 0, "anger": 4, "fear": 0, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 2, "joy": 73, "love": 2, "anger": 1, "fear": 1, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 0, "joy": 5, "love": 7, "anger": 0, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 2, "joy": 0, "love": 0, "anger": 26, "fear": 0, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 0, "joy": 0, "love": 0, "anger": 2, "fear": 18, "surprise": 0, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 125, "accuracy": 0.625, "wilson95": [ 0.5561430369897816, 0.6891456314159018 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 125, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.625, "brier": 0.5382340173, "nll": 0.7563524961565917, "ece10": 0.19070749999999997, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.5597428898476711, "confusion_matrix": { "false": { "false": 101, "true": 8, "__failed__": 0 }, "true": { "false": 67, "true": 24, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 152, "accuracy": 0.76, "wilson95": [ 0.6962665902079583, 0.8139338400758631 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 152, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.3696872093000001, "nll": 0.5585542188592619, "ece10": 0.05889249999999999, "calibration_n": 200, "confident_wrong": 0, "macro_f1": 0.75, "confusion_matrix": { "true": { "true": 56, "false": 21, "__failed__": 0 }, "false": { "true": 27, "false": 96, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.4696694742514195, "nll": 0.9130768919296007, "ece10": 0.25845427441495944, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.6787878787878787, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 2, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 1, "review": 1, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 2, "accuracy": 0.16666666666666666, "wilson95": [ 0.04696514218385378, 0.4480308622529734 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 2, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.16666666666666666, "brier": 0.8458638534561246, "nll": 1.7367333991984297, "ece10": 0.18660435210187687, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.10666666666666666, "confusion_matrix": { "maintenance": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 1, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 3, "it": 0, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.36543610065108334, "nll": 0.6862581590537328, "ece10": 0.2334686131386861, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.7267857142857144, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 1, "inspection": 1, "purchase": 1, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 1, "purchase": 0, "work": 2, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 6, "accuracy": 0.5, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 6, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6576113596242433, "nll": 1.2080504913384795, "ece10": 0.265135935816526, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.45, "confusion_matrix": { "design": { "design": 2, "inspection": 1, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 2, "purchase": 0, "work": 1, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 89, "accuracy": 0.89, "wilson95": [ 0.8136870349691969, 0.9374580364293543 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 89, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.89, "brier": 0.18437170670952166, "nll": 0.3955986578235469, "ece10": 0.13324820469523202, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8902017435452085, "confusion_matrix": { "label_000": { "label_000": 22, "label_001": 3, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 2, "label_001": 21, "label_002": 1, "label_003": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 3, "label_002": 1, "label_003": 21, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 71, "accuracy": 0.71, "wilson95": [ 0.6146110882442888, 0.7898516425087772 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 71, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.71, "brier": 0.3660230881474723, "nll": 0.6874096106175935, "ece10": 0.07978573491765018, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.679746342981637, "confusion_matrix": { "label_000": { "label_000": 16, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 1, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 17, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 7, "label_003": 10, "label_004": 0, "label_005": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 13, "label_004": 2, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 14, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 6, "label_002": 2, "label_003": 0, "label_004": 3, "label_005": 4, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 78, "accuracy": 0.78, "wilson95": [ 0.6892964648501421, 0.8499871761539459 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 78, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.78, "brier": 0.35561507972883205, "nll": 2.2717072144730626, "ece10": 0.1521723595199736, "calibration_n": 100, "confident_wrong": 10, "macro_f1": 0.7572089947089946, "confusion_matrix": { "label_000": { "label_000": 1, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 1, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 2, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 2, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 1, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 2, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 1, "label_021": 0, "label_022": 1, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 2, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 2, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 1, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 1, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 1, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 1, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 1, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 1, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.04403947552405435, "nll": 0.12740321876974417, "ece10": 0.11172158504563093, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 50, "accuracy": 0.45045045045045046, "wilson95": [ 0.36109608671838495, 0.5431196894224019 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 50, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45045045045045046, "brier": 0.706250968309286, "nll": 1.1991542334245393, "ece10": 0.18002993867865974, "calibration_n": 111, "confident_wrong": 0 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 46, "accuracy": 0.6388888888888888, "wilson95": [ 0.5235248046738358, 0.7401832029382592 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 46, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6388888888888888, "cluster_bootstrap95": [ 0.5, 0.7777777777777778 ], "independent_source_groups": 36, "brier": 0.5317890805094079, "nll": 0.996839888465217, "ece10": 0.17557376774175157, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 14, "accuracy": 0.7368421052631579, "wilson95": [ 0.5120844910218489, 0.8819358765321748 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7368421052631579, "brier": 0.39930934526315787, "nll": 0.5933919545745865, "ece10": 0.08787368421052631, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.6801346801346801, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.46859192000000005, "nll": 0.679279382194383, "ece10": 0.14284999999999998, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6862745098039216, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 3, "true": 5, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 18, "accuracy": 0.8181818181818182, "wilson95": [ 0.6148339256287345, 0.9269311541657794 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8181818181818182, "brier": 0.2967263281818182, "nll": 0.4726635734771403, "ece10": 0.012650000000000005, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.8166666666666667, "confusion_matrix": { "false": { "false": 8, "true": 2, "__failed__": 0 }, "true": { "false": 2, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.218986643799425, "nll": 0.42858483506177186, "ece10": 0.17521960585544608, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8106060606060606, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 1, "feature_request": 3, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 2, "praise": 0, "pricing_billing": 2, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.21669863194945888, "nll": 0.46417080053182097, "ece10": 0.10391696397378072, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8133333333333332, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 1, "4": 3, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.14441151715690956, "nll": 0.4529771751710048, "ece10": 0.12524087301040876, "calibration_n": 19, "confident_wrong": 1, "macro_f1": 0.9492063492063492, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 1, "roads": 3, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.5441594750113692, "nll": 1.1457665488384787, "ece10": 0.1788540396097757, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.6658730158730158, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 1, "performance": 1, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 2, "performance": 0, "security": 1, "style": 0, "refactor": 0, "nit": 1, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 1, "security": 0, "style": 1, "refactor": 0, "nit": 2, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 11, "accuracy": 0.7333333333333333, "wilson95": [ 0.4804956594401943, 0.8910254667430761 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7333333333333333, "brier": 0.47193481808484733, "nll": 1.1365674335482991, "ece10": 0.09588254740455877, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.6999999999999998, "confusion_matrix": { "feature": { "feature": 2, "fix": 0, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 1, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 1, "refactor": 0, "docs": 1, "test": 0, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.5767341411764706, "nll": 0.7970175697979851, "ece10": 0.23542352941176467, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5296442687747036, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 7, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 9, "accuracy": 0.5625, "wilson95": [ 0.331785563988119, 0.7690134759450765 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5625, "brier": 0.6148577937499999, "nll": 0.8450615713231586, "ece10": 0.25935624999999995, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.5607843137254902, "confusion_matrix": { "false": { "false": 5, "true": 3, "__failed__": 0 }, "true": { "false": 4, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.5457728103758258, "nll": 1.1864905723318633, "ece10": 0.1719503913345039, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.5349206349206349, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 2, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 2, "none": 1, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 1, "none": 5, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 15, "accuracy": 0.6818181818181818, "wilson95": [ 0.47318597741042745, 0.8363940681864375 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6818181818181818, "brier": 0.4669539034283616, "nll": 1.1102315574167854, "ece10": 0.17614304994274868, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.6039682539682539, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 1, "opinion": 3, "press_release": 1, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 1, "opinion": 0, "press_release": 0, "analysis": 3, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 2, "opinion": 1, "press_release": 0, "analysis": 1, "feature": 0, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.15132133936231412, "nll": 0.27807114492593016, "ece10": 0.08765745199445686, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.85, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 2, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 1, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 17, "accuracy": 0.8947368421052632, "wilson95": [ 0.6860591728889099, 0.9706414397011299 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8947368421052632, "brier": 0.13714365271023018, "nll": 0.2979467990258456, "ece10": 0.11567325674341679, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.8806349206349207, "confusion_matrix": { "address_issue": { "address_issue": 3, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 1, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 2, "failed_attempt": 0, "damaged": 1, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 10, "accuracy": 0.5882352941176471, "wilson95": [ 0.3600544264299978, 0.7838894320978518 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5882352941176471, "brier": 0.5887540823529411, "nll": 0.8175586384471324, "ece10": 0.23355294117647057, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5296442687747036, "confusion_matrix": { "false": { "false": 2, "true": 7, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 11, "accuracy": 0.7333333333333333, "wilson95": [ 0.4804956594401943, 0.8910254667430761 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7333333333333333, "brier": 0.40432737456315493, "nll": 0.7977006272921109, "ece10": 0.2016821797170218, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7047619047619047, "confusion_matrix": { "invoice": { "invoice": 1, "purchase_order": 1, "receipt": 1, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 1, "receipt": 1, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 1, "contract": 3, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 11, "accuracy": 0.7333333333333333, "wilson95": [ 0.4804956594401943, 0.8910254667430761 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7333333333333333, "brier": 0.33003734443092625, "nll": 0.6215744036438783, "ece10": 0.1644682109328154, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7295238095238096, "confusion_matrix": { "meeting_request": { "meeting_request": 1, "question": 1, "complaint": 0, "thanks": 1, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 1, "question": 0, "complaint": 2, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 1, "question": 0, "complaint": 0, "thanks": 0, "fyi": 2, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.13966082937148405, "nll": 0.30615634664887764, "ece10": 0.14041221645451107, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 4, "office_supplies": 0, "software": 0, "client_entertainment": 1, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 3, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.27106788666666665, "nll": 0.4385791450779254, "ece10": 0.0031111111111111014, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8328173374613003, "confusion_matrix": { "false": { "false": 7, "true": 2, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.8079256959162526, "nll": 1.6158893368731204, "ece10": 0.4155879324923793, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.46260683760683763, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 1, "1": 0, "2": 5, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.29158569089613534, "nll": 0.5498354654018542, "ece10": 0.24918411150889666, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.875, "confusion_matrix": { "0": { "0": 3, "1": 2, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 14, "accuracy": 0.7, "wilson95": [ 0.4810271816464765, 0.8545227551323956 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7, "brier": 0.4393814156053801, "nll": 0.9650985837297437, "ece10": 0.14535859031964704, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.6767032967032967, "confusion_matrix": { "cheating": { "cheating": 1, "harassment": 0, "boosting": 0, "smurfing": 1, "other": 2, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 3, "boosting": 0, "smurfing": 1, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 1, "other": 0, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 2, "other": 1, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 17, "accuracy": 0.8095238095238095, "wilson95": [ 0.5999943530979508, 0.9233243494723822 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8095238095238095, "brier": 0.21866175641232394, "nll": 0.40447871637847854, "ece10": 0.1516763267199061, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.8111111111111111, "confusion_matrix": { "spelling": { "spelling": 4, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 0, "punctuation": 0, "none": 1, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 4, "punctuation": 0, "none": 0, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 0, "punctuation": 4, "none": 0, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 2, "word_choice": 0, "punctuation": 0, "none": 3, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.38207557499999995, "nll": 0.5728019569654483, "ece10": 0.0820375, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.746031746031746, "confusion_matrix": { "false": { "false": 5, "true": 2, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 22, "accuracy": 0.9565217391304348, "wilson95": [ 0.7900884492974114, 0.9922833338565469 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9565217391304348, "brier": 0.12758432971774675, "nll": 0.29312150668178033, "ece10": 0.17658524894446118, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 1, "hvac": 0, "handyman": 0, "appliance_repair": 3, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 10, "accuracy": 0.7142857142857143, "wilson95": [ 0.4535091567010681, 0.8827862135542308 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7142857142857143, "brier": 0.390762107407145, "nll": 0.8379724532648473, "ece10": 0.08597148874814345, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.5314285714285714, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 1, "3": 1, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6571040174027446, "nll": 1.2180734157510518, "ece10": 0.2318462787879628, "calibration_n": 16, "confident_wrong": 2, "macro_f1": 0.4666666666666667, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 3, "1": 0, "2": 2, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 2, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.39820631814587565, "nll": 0.7114030425423715, "ece10": 0.21515105006382299, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7571428571428571, "confusion_matrix": { "0": { "0": 2, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.7307110282352941, "nll": 0.9818003765388793, "ece10": 0.3393764705882353, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.43956043956043955, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 7, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.24063045389235835, "nll": 0.5150059882590644, "ece10": 0.21200846274055524, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8806349206349205, "confusion_matrix": { "politics": { "politics": 3, "business": 0, "technology": 0, "sports": 0, "culture": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 1, "technology": 0, "sports": 2, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.3361754871416037, "nll": 0.716855438345584, "ece10": 0.31218510855790993, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8502164502164502, "confusion_matrix": { "payments": { "payments": 2, "web": 2, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 1, "infra": 3, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.7577768529411765, "nll": 1.0282756784394884, "ece10": 0.3638411764705883, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.32, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 9, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06630895125, "nll": 0.20082629798668045, "ece10": 0.18190625, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6106007386024855, "nll": 1.0533198859457134, "ece10": 0.042193132098556324, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.501994301994302, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 1, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 3, "__failed__": 0 }, "2": { "0": 2, "1": 1, "2": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.36634800081815544, "nll": 0.6989902938658109, "ece10": 0.20159991906819996, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7222222222222223, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 2, "french": 1, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 1, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 2, "indian": 0, "thai": 0, "middle_eastern": 1, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 1, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.2881232870588235, "nll": 0.46167924684225137, "ece10": 0.007788235294117638, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8210526315789474, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 12, "accuracy": 0.6666666666666666, "wilson95": [ 0.43749467295945105, 0.837212252491663 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.4127259220666609, "nll": 0.7045839172329601, "ece10": 0.15730899664853443, "calibration_n": 18, "confident_wrong": 1, "macro_f1": 0.6133333333333333, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 4, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 3, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 1, "size_or_fit": 2, "changed_mind": 1, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 9, "accuracy": 0.5, "wilson95": [ 0.29031021505169974, 0.7096897849483003 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.6604716541242053, "nll": 1.4397899228976772, "ece10": 0.11289000235017783, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.4666666666666667, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 1, "3": 2, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 1, "1": 0, "2": 0, "3": 2, "4": 1, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 11, "accuracy": 0.6875, "wilson95": [ 0.44404355834740666, 0.8583535614521796 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6875, "brier": 0.45880866249999996, "nll": 0.662407979920124, "ece10": 0.1321375, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6135265700483092, "confusion_matrix": { "false": { "false": 9, "true": 1, "__failed__": 0 }, "true": { "false": 4, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 10, "accuracy": 0.6666666666666666, "wilson95": [ 0.41713547738519774, 0.8482367556028526 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6666666666666666, "brier": 0.48561133999999995, "nll": 0.6942386401212556, "ece10": 0.14502, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.660633484162896, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 3, "true": 4, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.266862570911421, "nll": 0.5476375026993682, "ece10": 0.09915115316433505, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8076190476190476, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 2, "account": 0, "other": 2, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 8, "accuracy": 0.47058823529411764, "wilson95": [ 0.26165106442409436, 0.6903676493999558 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47058823529411764, "brier": 0.7445430070588236, "nll": 1.0051574366386655, "ece10": 0.35165882352941175, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.3952569169960475, "confusion_matrix": { "false": { "false": 1, "true": 7, "__failed__": 0 }, "true": { "false": 2, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.007149164282324832, "nll": 0.06501579890989873, "ece10": 0.06214147015573049, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 11, "accuracy": 0.6470588235294118, "wilson95": [ 0.41300363485218694, 0.8269027960275627 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6470588235294118, "brier": 0.5178903329411765, "nll": 0.7358785168826015, "ece10": 0.1754058823529412, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.39285714285714285, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 6, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 11, "accuracy": 0.6111111111111112, "wilson95": [ 0.3861904160223791, 0.7969475342783638 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6111111111111112, "brier": 0.5502428588888889, "nll": 0.7674582526465108, "ece10": 0.21423888888888884, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5785953177257526, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 7, "true": 3, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 11, "accuracy": 0.55, "wilson95": [ 0.34208534245034233, 0.7418021417443759 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.55, "brier": 0.6516996317208638, "nll": 1.1397836464119435, "ece10": 0.2265047608486976, "calibration_n": 20, "confident_wrong": 1, "macro_f1": 0.5730042016806722, "confusion_matrix": { "0": { "0": 1, "1": 6, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 3, "2": 0, "3": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 18, "accuracy": 0.9, "wilson95": [ 0.6989663547715128, 0.9721335187862319 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.14057673833933446, "nll": 0.19974034147846656, "ece10": 0.09104996424774964, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9024531024531024, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 1, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 1, "reminder": 0, "music": 0, "small_talk": 3, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 9, "accuracy": 0.45, "wilson95": [ 0.2581978582556243, 0.6579146575496577 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.45, "brier": 0.7848066929999999, "nll": 1.0604589625594039, "ece10": 0.38490499999999994, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.3103448275862069, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 11, "true": 0, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 16, "accuracy": 0.7272727272727273, "wilson95": [ 0.5184826598759762, 0.8684923971201053 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7272727272727273, "brier": 0.4792353569489273, "nll": 2.335438036442795, "ece10": 0.26837425746942434, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.718939393939394, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 1, "3": 0, "__failed__": 0 }, "1": { "0": 1, "1": 5, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 3, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 389, "accuracy": 0.43222222222222223, "wilson95": [ 0.4002133029601228, 0.4648072724855067 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 519, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.483690587138863, "cluster_bootstrap95": [ 0.3821133085227448, 0.47703601026669923 ], "independent_source_groups": 596, "brier": 0.6796302205025694, "nll": 1.3321555786476327, "ece10": 0.2033319332305434, "calibration_n": 1073, "confident_wrong": 9 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 229, "accuracy": 0.3989547038327526, "wilson95": [ 0.3596944903331517, 0.43955840467936225 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 525, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5608974358974359, "brier": 0.6174651201683631, "nll": 1.2905090155844718, "ece10": 0.14217543522341805, "calibration_n": 936, "confident_wrong": 1 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 141, "accuracy": 0.20142857142857143, "wilson95": [ 0.17338359478973164, 0.2327326622713819 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 368, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.3382352941176471, "cluster_bootstrap95": [ 0.17171608791780466, 0.2301218196366222 ], "independent_source_groups": 650, "brier": 0.7555483327063278, "nll": 1.390804499864913, "ece10": 0.08984615064366985, "calibration_n": 1088, "confident_wrong": 19 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 5, "accuracy": 0.0125, "wilson95": [ 0.005350750779971611, 0.028923736195967484 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 826, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.413, "brier": 0.6759698492853061, "nll": 1.185087779619007, "ece10": 0.09028697650240958, "calibration_n": 2000, "confident_wrong": 0 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "wilson95": [ 0.7735463743160095, 0.9801879358846942 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 26, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.18260728445446478, "nll": 0.40227261999955466, "ece10": 0.22735854741587114, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.925, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 1, "inspection": 1, "purchase": 0, "work": 5, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 26, "accuracy": 0.9285714285714286, "wilson95": [ 0.7735463743160095, 0.9801879358846942 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 26, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9285714285714286, "brier": 0.1826072844544648, "nll": 0.40227261999955477, "ece10": 0.22735854741587117, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.925, "confusion_matrix": { "work": { "work": 5, "purchase": 0, "inspection": 1, "design": 1, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 12, "accuracy": 0.42857142857142855, "wilson95": [ 0.26508547124123694, 0.6092921437253123 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42857142857142855, "brier": 0.6706904218548996, "nll": 1.265425770234938, "ece10": 0.203218547609614, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.29647058823529415, "confusion_matrix": { "design": { "design": 6, "inspection": 1, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 1, "inspection": 6, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "work": { "design": 4, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 12, "accuracy": 0.42857142857142855, "wilson95": [ 0.26508547124123694, 0.6092921437253123 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42857142857142855, "brier": 0.6706904218548996, "nll": 1.265425770234938, "ece10": 0.20321854760961405, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.29647058823529415, "confusion_matrix": { "work": { "work": 0, "purchase": 0, "inspection": 3, "design": 4, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 6, "design": 1, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 1, "design": 6, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 34, "accuracy": 0.85, "wilson95": [ 0.709276756335103, 0.9293881228267964 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.23675933030686744, "nll": 0.494525387574445, "ece10": 0.11302673843379236, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8335353535353536, "confusion_matrix": { "review": { "review": 8, "it": 0, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 2, "it": 0, "documents": 3, "procurement": 3, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 1, "procurement": 0, "maintenance": 7, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 34, "accuracy": 0.85, "wilson95": [ 0.709276756335103, 0.9293881228267964 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.23675933030686744, "nll": 0.494525387574445, "ece10": 0.11302673843379238, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8335353535353536, "confusion_matrix": { "maintenance": { "maintenance": 7, "procurement": 0, "documents": 1, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 3, "documents": 3, "it": 0, "review": 2, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 8, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 15, "accuracy": 0.375, "wilson95": [ 0.24222979167233266, 0.5296756086269889 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.375, "brier": 0.8351649505184889, "nll": 1.7749297311698964, "ece10": 0.2605976930393269, "calibration_n": 40, "confident_wrong": 1, "macro_f1": 0.25656565656565655, "confusion_matrix": { "review": { "review": 7, "it": 0, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 6, "it": 1, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 1, "it": 0, "documents": 7, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 4, "it": 0, "documents": 4, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 7, "it": 0, "documents": 1, "procurement": 0, "maintenance": 0, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 15, "accuracy": 0.375, "wilson95": [ 0.24222979167233266, 0.5296756086269889 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.375, "brier": 0.8351649505184892, "nll": 1.7749297311698964, "ece10": 0.26059769303932695, "calibration_n": 40, "confident_wrong": 1, "macro_f1": 0.25656565656565655, "confusion_matrix": { "maintenance": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 7, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 0, "documents": 4, "it": 0, "review": 4, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 7, "it": 0, "review": 1, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 1, "review": 6, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 7, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 355, "accuracy": 0.8875, "wilson95": [ 0.8527764306768796, 0.9148515412140404 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 355, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8875, "brier": 0.20913888113064824, "nll": 0.4307699724523479, "ece10": 0.13956186383207678, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.7632643164588803, "confusion_matrix": { "sadness": { "sadness": 118, "joy": 0, "love": 0, "anger": 1, "fear": 0, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 2, "joy": 109, "love": 6, "anger": 0, "fear": 1, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 5, "joy": 15, "love": 16, "anger": 1, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 2, "joy": 0, "love": 0, "anger": 62, "fear": 0, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 0, "joy": 0, "love": 0, "anger": 4, "fear": 48, "surprise": 0, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 3, "love": 0, "anger": 0, "fear": 3, "surprise": 2, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 237, "accuracy": 0.79, "wilson95": [ 0.7403888774230657, 0.8322781992666975 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 237, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.3842732710059544, "nll": 3.9765390184959006, "ece10": 0.16823053695421872, "calibration_n": 300, "confident_wrong": 45 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 292, "accuracy": 0.73, "wilson95": [ 0.684457128728908, 0.7711672158773156 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 292, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.73, "brier": 0.40858604249999997, "nll": 0.6048011056073471, "ece10": 0.08920149999999996, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.6575342465753424, "confusion_matrix": { "false": { "false": 238, "true": 71, "__failed__": 0 }, "true": { "false": 37, "true": 54, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 346, "accuracy": 0.865, "wilson95": [ 0.8280190157519688, 0.8950370093839947 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 346, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.865, "brier": 0.2547359276967978, "nll": 0.5298409301276618, "ece10": 0.16946768805352685, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.8477453650151018, "confusion_matrix": { "world": { "world": 87, "sports": 6, "business": 4, "sci_tech": 6, "__failed__": 0 }, "sports": { "world": 2, "sports": 118, "business": 0, "sci_tech": 3, "__failed__": 0 }, "business": { "world": 8, "sports": 0, "business": 46, "sci_tech": 18, "__failed__": 0 }, "sci_tech": { "world": 3, "sports": 0, "business": 4, "sci_tech": 95, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 278, "accuracy": 0.695, "wilson95": [ 0.6482028147696431, 0.7380873904399811 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 278, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.695, "brier": 0.45069416569999987, "nll": 0.6535320323265554, "ece10": 0.12469000000000001, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.6746666666666667, "confusion_matrix": { "true": { "true": 89, "false": 64, "__failed__": 0 }, "false": { "true": 58, "false": 189, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 161, "accuracy": 0.5366666666666666, "wilson95": [ 0.4801320652390865, 0.5922741168711132 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 161, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5366666666666666, "brier": 0.8124873979293609, "nll": 7.688277405046547, "ece10": 0.3751154023848339, "calibration_n": 300, "confident_wrong": 92 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 235, "accuracy": 0.5875, "wilson95": [ 0.5386474925901331, 0.634687855901365 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 235, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5875, "brier": 0.5879671625999996, "nll": 0.8161298579955155, "ece10": 0.22739050000000002, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.5530847310721226, "confusion_matrix": { "false": { "false": 173, "true": 15, "__failed__": 0 }, "true": { "false": 150, "true": 62, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 274, "accuracy": 0.685, "wilson95": [ 0.637901366761192, 0.7285790843351184 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 274, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.685, "brier": 0.4658980351500005, "nll": 0.6728329136312516, "ece10": 0.13856325000000008, "calibration_n": 400, "confident_wrong": 0, "macro_f1": 0.6843608306821314, "confusion_matrix": { "false": { "false": 128, "true": 72, "__failed__": 0 }, "true": { "false": 54, "true": 146, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 57, "accuracy": 0.57, "wilson95": [ 0.47215389549212305, 0.6626670147588988 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 57, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.57, "brier": 0.4919792144000003, "nll": 0.6854628939132921, "ece10": 0.04298599999999999, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.5647332725984411, "confusion_matrix": { "yes": { "yes": 23, "no": 31, "__failed__": 0 }, "no": { "yes": 12, "no": 34, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 34, "accuracy": 0.34, "wilson95": [ 0.25461520797348164, 0.43722271145275377 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.34, "brier": 1.1678094737008045, "nll": 13.066019088912297, "ece10": 0.5644473765435685, "calibration_n": 100, "confident_wrong": 41, "macro_f1": 0.3250588227561912, "confusion_matrix": { "balance": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 2, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 1, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 1, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 1, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 3, "mortgage": 0, "wire": 0, "statement": 2, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 0, "wire": 1, "statement": 4, "fraud_alert": 0, "branch_appointment": 2, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 2, "statement": 4, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 4, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 1, "fraud_alert": 0, "branch_appointment": 5, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 4, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 1, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 5, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 1, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 1, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 2, "fraud_alert": 1, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 2, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 1, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 2, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 3, "close_account": 0, "new_account": 0, "limit_increase": 3, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 0, "card_lost": 2, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 4, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 34, "accuracy": 0.34, "wilson95": [ 0.25461520797348164, 0.43722271145275377 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 124, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.62, "brier": 0.4833254078321649, "nll": 0.8626206098768922, "ece10": 0.16428354000354642, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 17, "accuracy": 0.17, "wilson95": [ 0.10893566114059931, 0.25548004767601123 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 83, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.415, "brier": 0.6707791703983855, "nll": 1.376498320721391, "ece10": 0.09592094694086613, "calibration_n": 200, "confident_wrong": 0 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48, "brier": 0.6822077566898089, "nll": 1.6684361721444256, "ece10": 0.12002658776109315, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.47076860639360635, "confusion_matrix": { "invoice": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 1, "clinical_note": 1, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 2, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 6, "contract": 1, "resume": 0, "bank_statement": 1, "clinical_note": 2, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 3, "recipe": 1, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 2, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 2, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 2, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 9, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 4, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 7, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 3, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 4, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 2, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 0, "receipt": 0, "contract": 3, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 3, "research_abstract": 2, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 1, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 1, "lease": 0, "research_abstract": 5, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 3, "sports_recap": 1, "__failed__": 0 }, "sports_recap": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 2, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 5, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 1, "accuracy": 0.01, "wilson95": [ 0.0017674320641406505, 0.054486196178705315 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 161, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4025, "brier": 0.6581639006420982, "nll": 1.1914640790412379, "ece10": 0.0543047349066387, "calibration_n": 400, "confident_wrong": 0 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 34, "accuracy": 0.34, "wilson95": [ 0.25461520797348164, 0.43722271145275377 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.34, "brier": 0.7468869735568049, "nll": 1.6900300766522078, "ece10": 0.09116284737947941, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.3348150306916085, "confusion_matrix": { "politics": { "politics": 3, "business": 0, "technology": 1, "science": 7, "health": 2, "sports": 0, "entertainment": 0, "world": 0, "culture": 1, "weather": 0, "__failed__": 0 }, "business": { "politics": 5, "business": 1, "technology": 0, "science": 3, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 6, "science": 3, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 3, "business": 0, "technology": 0, "science": 7, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "health": { "politics": 2, "business": 0, "technology": 0, "science": 1, "health": 7, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 1, "business": 0, "technology": 0, "science": 2, "health": 1, "sports": 4, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 1, "technology": 2, "science": 1, "health": 0, "sports": 0, "entertainment": 3, "world": 0, "culture": 1, "weather": 1, "__failed__": 0 }, "world": { "politics": 0, "business": 2, "technology": 1, "science": 4, "health": 1, "sports": 0, "entertainment": 1, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 2, "business": 0, "technology": 1, "science": 3, "health": 2, "sports": 0, "entertainment": 0, "world": 0, "culture": 2, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 1, "science": 5, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 1, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 25, "accuracy": 0.25, "wilson95": [ 0.17545211362287672, 0.34304463548061603 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 25, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.25, "brier": 0.8403341372799816, "nll": 2.0623269790806895, "ece10": 0.11267667622722458, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.24, "confusion_matrix": { "biology": { "biology": 6, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 3, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 5, "medicine": 2, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 2, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 2, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 1, "mathematics": 2, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 4, "medicine": 0, "chemistry": 0, "physics": 6, "computer_science": 1, "mathematics": 5, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 0, "chemistry": 2, "physics": 4, "computer_science": 2, "mathematics": 6, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 2, "computer_science": 1, "mathematics": 3, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 6, "medicine": 0, "chemistry": 1, "physics": 1, "computer_science": 0, "mathematics": 2, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 2, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 3, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 4, "earth_science": 0, "engineering": 0, "economics": 4, "other": 0, "__failed__": 0 }, "other": { "biology": 2, "medicine": 0, "chemistry": 0, "physics": 1, "computer_science": 0, "mathematics": 8, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 63, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.315, "brier": 0.5827003691260191, "nll": 1.0763564816308893, "ece10": 0.17964277054953764, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 13, "accuracy": 0.13, "wilson95": [ 0.07757167427240512, 0.20980351440076428 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 84, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42, "brier": 0.4331931378940451, "nll": 0.7616999978502839, "ece10": 0.20634748370531486, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 68, "accuracy": 0.68, "wilson95": [ 0.5833738138954762, 0.7633085267500092 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 68, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.68, "brier": 0.42424463753721153, "nll": 0.7557515389219092, "ece10": 0.15375366723380665, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.6023230353127261, "confusion_matrix": { "negative": { "negative": 36, "neutral": 0, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 22, "neutral": 4, "positive": 7, "__failed__": 0 }, "positive": { "negative": 3, "neutral": 0, "positive": 28, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 64, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.32, "brier": 0.4399032568000003, "nll": 0.6315116602700178, "ece10": 0.10584600000000002, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 20, "accuracy": 0.2, "wilson95": [ 0.1333669333310325, 0.2888291655931589 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 183, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.5628455203349393, "nll": 1.0275272764754233, "ece10": 0.18219016813972785, "calibration_n": 300, "confident_wrong": 0 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 34, "accuracy": 0.34, "wilson95": [ 0.25461520797348164, 0.43722271145275377 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.34, "brier": 1.1408337828968964, "nll": 13.90593982637031, "ece10": 0.5175929580093797, "calibration_n": 100, "confident_wrong": 36, "macro_f1": 0.33580921879241205, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 3, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 2, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 3, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 2, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 2, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 1, "speak_to_human": 2, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 1, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 4, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 3, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 1, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 2, "speak_to_human": 2, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 2, "complaint": 0, "invoice_copy": 0, "tax_document": 2, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 4, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 1, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 2, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 1, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 2, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 21, "accuracy": 0.21, "wilson95": [ 0.141656540619153, 0.29979968834089865 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.21, "brier": 0.9560175070503295, "nll": 2.4333395619971636, "ece10": 0.2791836188907509, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.18510003885003887, "confusion_matrix": { "billing": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 6, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 2, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 6, "onboarding": 0, "integrations": 0, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 6, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 3, "technical_outage": 4, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 1, "account_access": 0, "shipping": 1, "returns": 0, "technical_outage": 8, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 7, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 9, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 3, "onboarding": 0, "integrations": 0, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 2, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 1, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 3, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 2, "data_privacy": 1, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 5, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "other": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 0, "technical_outage": 5, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 1, "data_privacy": 1, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 93, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.31, "brier": 0.930153070159497, "nll": 4.355181800345136, "ece10": 0.303282504113835, "calibration_n": 300, "confident_wrong": 50 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 34, "accuracy": 0.34, "wilson95": [ 0.25461520797348164, 0.43722271145275377 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 34, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.34, "brier": 0.8431587120989977, "nll": 2.123277242190666, "ece10": 0.20476466930960202, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.3540270350564468, "confusion_matrix": { "new_booking": { "new_booking": 2, "change_flight": 0, "cancel_trip": 1, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 1, "disruption": 1, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 2, "change_flight": 2, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 7, "other": 0, "__failed__": 0 }, "cancel_trip": { "new_booking": 1, "change_flight": 0, "cancel_trip": 3, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 2, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 4, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 3, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 7, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 3, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 3, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 3, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 2, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 4, "car": 0, "visa_docs": 0, "loyalty": 1, "disruption": 4, "other": 0, "__failed__": 0 }, "car": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 1, "__failed__": 0 }, "visa_docs": { "new_booking": 2, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 3, "loyalty": 1, "disruption": 2, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 3, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 1, "loyalty": 2, "disruption": 3, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 1, "disruption": 6, "other": 0, "__failed__": 0 }, "other": { "new_booking": 2, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 2, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 2, "disruption": 6, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 89, "accuracy": 0.89, "wilson95": [ 0.8136870349691969, 0.9374580364293543 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 89, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.89, "brier": 0.3185682879927025, "nll": 0.6149396990594652, "ece10": 0.3018226400561764, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.8900709219858156, "confusion_matrix": { "business": { "business": 23, "science and technology": 2, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 2, "science and technology": 21, "sports": 1, "world": 1, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 1, "sports": 24, "world": 0, "__failed__": 0 }, "world": { "business": 1, "science and technology": 2, "sports": 1, "world": 21, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 74, "accuracy": 0.74, "wilson95": [ 0.6462901055081282, 0.8159530153525186 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 74, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.74, "brier": 0.43040899456667403, "nll": 3.331424605806951, "ece10": 0.20712492537852928, "calibration_n": 100, "confident_wrong": 12, "macro_f1": 0.7345238095238095, "confusion_matrix": { "refund not showing up": { "refund not showing up": 1, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 1, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 1, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 1, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 2, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 1, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 1, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 2, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 2, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 2, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 1, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 1, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 1, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 1, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 1, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 1, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 1, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 1, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 76, "accuracy": 0.76, "wilson95": [ 0.6676766365018553, 0.8330867444305122 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 76, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.35306681907050574, "nll": 0.6667589276036412, "ece10": 0.06623974738552194, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.7263232553480231, "confusion_matrix": { "anger": { "anger": 16, "fear": 0, "joy": 0, "love": 0, "sadness": 1, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 0, "fear": 17, "joy": 0, "love": 0, "sadness": 0, "surprise": 0, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 16, "love": 1, "sadness": 0, "surprise": 0, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 7, "love": 8, "sadness": 2, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 1, "joy": 0, "love": 0, "sadness": 15, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 5, "joy": 2, "love": 0, "sadness": 4, "surprise": 4, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 1665, "accuracy": 0.5647896879240163, "wilson95": [ 0.5468199427314454, 0.5825908014867128 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 1665, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5647896879240163, "brier": 0.8333613940173239, "nll": 11.442371339841191, "ece10": 0.4053333285039084, "calibration_n": 2948, "confident_wrong": 1110, "macro_f1": 0.6153029702332901, "confusion_matrix": { "alarm": { "alarm": 72, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 11, "email": 0, "general": 7, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 41, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 16, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 2, "audio": 0, "calendar": 198, "cooking": 0, "datetime": 110, "email": 2, "general": 65, "iot": 0, "lists": 10, "music": 0, "news": 0, "play": 0, "qa": 4, "recommendation": 8, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 52, "datetime": 1, "email": 0, "general": 7, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 11, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 85, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 10, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 4, "email": 209, "general": 43, "iot": 0, "lists": 11, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 3, "cooking": 2, "datetime": 8, "email": 2, "general": 129, "iot": 2, "lists": 2, "music": 1, "news": 0, "play": 3, "qa": 20, "recommendation": 6, "social": 0, "takeaway": 2, "transport": 5, "weather": 1, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 24, "calendar": 0, "cooking": 0, "datetime": 5, "email": 0, "general": 158, "iot": 22, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 0, "datetime": 0, "email": 2, "general": 28, "iot": 0, "lists": 103, "music": 1, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 2, "email": 0, "general": 43, "iot": 1, "lists": 4, "music": 29, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 0, "datetime": 3, "email": 7, "general": 24, "iot": 0, "lists": 0, "music": 0, "news": 76, "play": 1, "qa": 10, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 17, "calendar": 0, "cooking": 0, "datetime": 2, "email": 5, "general": 91, "iot": 0, "lists": 0, "music": 91, "news": 1, "play": 176, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 21, "email": 0, "general": 97, "iot": 0, "lists": 3, "music": 0, "news": 1, "play": 0, "qa": 157, "recommendation": 7, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 9, "email": 0, "general": 22, "iot": 0, "lists": 2, "music": 2, "news": 1, "play": 3, "qa": 2, "recommendation": 50, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 7, "general": 25, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 73, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 3, "datetime": 4, "email": 1, "general": 12, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 35, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 31, "email": 0, "general": 17, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 6, "social": 0, "takeaway": 0, "transport": 65, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 5, "email": 0, "general": 36, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 15, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 3, "weather": 93, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 394, "accuracy": 0.13364993215739485, "wilson95": [ 0.12184212154942771, 0.14641126192904913 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 394, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.13364993215739485, "brier": 1.7214163643884974, "nll": 28.70336315172301, "ece10": 0.8586043472838104, "calibration_n": 2948, "confident_wrong": 2501, "macro_f1": 0.14451148002210393, "confusion_matrix": { "alarm": { "alarm": 33, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 61, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 60, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 5, "email": 7, "general": 386, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 7, "datetime": 0, "email": 0, "general": 60, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 2, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 98, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 71, "general": 197, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 1, "general": 180, "iot": 0, "lists": 1, "music": 1, "news": 0, "play": 1, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 213, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 138, "iot": 0, "lists": 3, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 69, "iot": 0, "lists": 0, "music": 12, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 1, "general": 122, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 367, "iot": 0, "lists": 0, "music": 14, "news": 0, "play": 4, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 281, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 4, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 92, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 5, "general": 42, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 58, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 54, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 2, "general": 100, "iot": 0, "lists": 2, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 19, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 151, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 582, "accuracy": 0.4961636828644501, "wilson95": [ 0.4676104065745571, 0.5247420042514357 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 582, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4961636828644501, "cluster_bootstrap95": [ 0.4515090976656093, 0.5369073645994173 ], "independent_source_groups": 47, "brier": 0.625428845923415, "nll": 1.0527953730667419, "ece10": 0.14132859624241978, "calibration_n": 1173, "confident_wrong": 4, "macro_f1": 0.29641656646476844, "confusion_matrix": { "negative": { "negative": 15, "neutral": 165, "positive": 2, "__failed__": 0 }, "neutral": { "negative": 30, "neutral": 542, "positive": 18, "__failed__": 0 }, "positive": { "negative": 21, "neutral": 355, "positive": 25, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "wald-4b-v12": { "label": "Wald 4B v1.2", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 170, "accuracy": 0.85, "wilson95": [ 0.7939442071583334, 0.89286406437758 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 170, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.2105234104688268, "nll": 0.38672063339588, "ece10": 0.04747295313772787, "calibration_n": 200, "confident_wrong": 3, "macro_f1": 0.8523368926093831, "confusion_matrix": { "world": { "world": 43, "sports": 0, "business": 3, "sci_tech": 4, "__failed__": 0 }, "sports": { "world": 0, "sports": 46, "business": 1, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 2, "sports": 0, "business": 47, "sci_tech": 2, "__failed__": 0 }, "sci_tech": { "world": 3, "sports": 0, "business": 15, "sci_tech": 34, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 121, "accuracy": 0.605, "wilson95": [ 0.5358834915928536, 0.6701589898679203 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 121, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.605, "brier": 0.5854746260517231, "nll": 1.1887049295683472, "ece10": 0.14442974863728883, "calibration_n": 200, "confident_wrong": 11, "macro_f1": 0.48948719216128217, "confusion_matrix": { "sadness": { "sadness": 43, "joy": 3, "love": 1, "anger": 5, "fear": 5, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 19, "joy": 49, "love": 6, "anger": 3, "fear": 2, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 2, "joy": 6, "love": 3, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 }, "anger": { "sadness": 7, "joy": 3, "love": 1, "anger": 15, "fear": 2, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 3, "joy": 2, "love": 2, "anger": 2, "fear": 10, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 184, "accuracy": 0.92, "wilson95": [ 0.874010511732856, 0.9501594141102401 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 184, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.92, "brier": 0.15395676041983408, "nll": 0.26172026194755804, "ece10": 0.05422022026822851, "calibration_n": 200, "confident_wrong": 2, "macro_f1": 0.9176191947276284, "confusion_matrix": { "false": { "false": 109, "true": 0, "__failed__": 0 }, "true": { "false": 16, "true": 75, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 181, "accuracy": 0.905, "wilson95": [ 0.8563983883518006, 0.9383368972826135 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 181, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.905, "brier": 0.15062058872951745, "nll": 0.25172339276214184, "ece10": 0.041768672035704855, "calibration_n": 200, "confident_wrong": 4, "macro_f1": 0.8959445767956407, "confusion_matrix": { "true": { "true": 61, "false": 16, "__failed__": 0 }, "false": { "true": 3, "false": 120, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 10, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.19294020376472062, "nll": 0.3756617201114513, "ece10": 0.08880231595957644, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.8533333333333333, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 1, "it": 0, "review": 2, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3784559996521304, "nll": 0.7929225036363344, "ece10": 0.2458476821420817, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.7733333333333332, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 2, "review": 1, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.013011119266387934, "nll": 0.07952364695942034, "ece10": 0.07517226561486656, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.01585751760434633, "nll": 0.09117904972704928, "ece10": 0.08544894420935822, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 83, "accuracy": 0.83, "wilson95": [ 0.7445199523239887, 0.8910643388594006 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 83, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.83, "brier": 0.22129112520857713, "nll": 0.3917590061284997, "ece10": 0.0854594946937349, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.8005719154442794, "confusion_matrix": { "label_000": { "label_000": 25, "label_001": 0, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 13, "label_001": 8, "label_002": 1, "label_003": 3, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 43, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43, "brier": 0.754425685474273, "nll": 1.5134784989195913, "ece10": 0.27220345126531353, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.43809222730791353, "confusion_matrix": { "label_000": { "label_000": 8, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 7, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 1, "label_001": 7, "label_002": 0, "label_003": 0, "label_004": 8, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 9, "label_003": 1, "label_004": 5, "label_005": 2, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 3, "label_003": 5, "label_004": 9, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 2, "label_002": 2, "label_003": 1, "label_004": 11, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 0, "label_002": 3, "label_003": 2, "label_004": 7, "label_005": 3, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 84, "accuracy": 0.84, "wilson95": [ 0.7557973061072976, 0.8990471151119521 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 84, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.84, "brier": 0.24577066083771548, "nll": 0.52534408214467, "ece10": 0.08558354442787919, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.8111111111111111, "confusion_matrix": { "label_000": { "label_000": 2, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_001": { "label_000": 0, "label_001": 2, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 2, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 1, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 2, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 2, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 2, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 2, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 2, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 1, "label_011": 0, "label_012": 0, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 2, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 1, "label_013": 1, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 2, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 2, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 2, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 2, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 2, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 2, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 1, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 2, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 2, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 2, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 1, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 1, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 1, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 1, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 2, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 2, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 1, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 1, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 1, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 1, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 1, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 1, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 1, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 1, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 1, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 1, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 1, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 1, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 1, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 1, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 1, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 1, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 1, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 1, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 1, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 1, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 1, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 1, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 1, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 1, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 1, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 1, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 1, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 1, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 1, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 1, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 1, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 1, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 1, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 1, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 1, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 1, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 1, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 1, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 1, "label_070": 0, "label_071": 0, "__failed__": 0 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 1, "label_071": 0, "__failed__": 0 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 1, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.000865882565921881, "nll": 0.010643808775336232, "ece10": 0.010354728229121627, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 77, "accuracy": 0.6936936936936937, "wilson95": [ 0.6026596151427787, 0.7717696235796009 ], "valid": 101, "failures": 10, "coverage": 0.9099099099099099, "heads": 111, "head_correct": 77, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6936936936936937, "brier": 0.3693820565891312, "nll": 0.6929008565759455, "ece10": 0.10775830710790069, "calibration_n": 101, "confident_wrong": 3 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 72, "accuracy": 1.0, "wilson95": [ 0.9493488274035422, 1.0 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 72, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "cluster_bootstrap95": [ 1.0, 1.0 ], "independent_source_groups": 36, "brier": 0.022650099055260206, "nll": 0.089528365409459, "ece10": 0.08149483541144155, "calibration_n": 72, "confident_wrong": 0 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 18, "accuracy": 0.9473684210526315, "wilson95": [ 0.753612688331012, 0.9906480059376995 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9473684210526315, "brier": 0.13284848841130933, "nll": 0.25071938633778307, "ece10": 0.12583515818787425, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.9449275362318841, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.060919003233774735, "nll": 0.13504262853535356, "ece10": 0.11534617244381082, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 21, "accuracy": 0.9545454545454546, "wilson95": [ 0.7820195416134267, 0.991930572378736 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9545454545454546, "brier": 0.11393133976930835, "nll": 0.23709328588461007, "ece10": 0.1391420077082572, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9536842105263157, "confusion_matrix": { "false": { "false": 9, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.014101254567720442, "nll": 0.07159618440680779, "ece10": 0.0669312935981915, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 17, "accuracy": 0.9444444444444444, "wilson95": [ 0.7424269921812741, 0.990124809021697 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9444444444444444, "brier": 0.1715655653600445, "nll": 0.376691684614834, "ece10": 0.23362169253369286, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 1, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.01096509031473085, "nll": 0.03637653577322823, "ece10": 0.03268715304443687, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.19888142736380066, "nll": 0.3567371531150186, "ece10": 0.09724075547793702, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9027777777777778, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 3, "refactor": 0, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 2, "refactor": 0, "nit": 2, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.09498470952029169, "nll": 0.19193216894938114, "ece10": 0.08087138977507503, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9111111111111111, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 2, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 2, "test": 0, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 1, "refactor": 0, "docs": 0, "test": 1, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.045154748530713246, "nll": 0.13185532797419675, "ece10": 0.11788293099295974, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.2119697447319556, "nll": 0.33722518551649755, "ece10": 0.06439783181440212, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.873015873015873, "confusion_matrix": { "false": { "false": 6, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.1937240954470666, "nll": 0.4131874138990866, "ece10": 0.17672186404705192, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.877948717948718, "confusion_matrix": { "harassment": { "harassment": 4, "threat": 0, "self_harm": 0, "spam": 0, "none": 1, "__failed__": 0 }, "threat": { "harassment": 1, "threat": 1, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 6, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.1336024799624562, "nll": 0.28472614884295083, "ece10": 0.12410261795161767, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 2, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 2, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.004599986975286476, "nll": 0.027737050986235714, "ece10": 0.02636000832423396, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0009171299366310464, "nll": 0.016969158621695877, "ece10": 0.016681739982671265, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 15, "accuracy": 0.8823529411764706, "wilson95": [ 0.6566364940511412, 0.9671202262362074 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8823529411764706, "brier": 0.13405498814250089, "nll": 0.2572481933665712, "ece10": 0.2084067122699282, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8819444444444444, "confusion_matrix": { "false": { "false": 7, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0012705107936737826, "nll": 0.017319780809761275, "ece10": 0.016898126550241054, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 12, "accuracy": 0.8, "wilson95": [ 0.5481455128483065, 0.9295245065301844 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.1811264925241814, "nll": 0.2763612343057889, "ece10": 0.1390768940544669, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.7814285714285714, "confusion_matrix": { "meeting_request": { "meeting_request": 2, "question": 1, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 1, "thanks": 0, "fyi": 2, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.05080619856256839, "nll": 0.11610598358841957, "ece10": 0.09517382701758757, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 3, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 15, "accuracy": 0.8333333333333334, "wilson95": [ 0.6077796189608428, 0.9416342319413856 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8333333333333334, "brier": 0.21279470940680553, "nll": 0.3374185081362494, "ece10": 0.07523611651921505, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8285714285714285, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 8, "accuracy": 0.5, "wilson95": [ 0.27999563610326017, 0.7200043638967398 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 8, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.47945145774425396, "nll": 0.7411547163933916, "ece10": 0.24068886380113907, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.4875, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 6, "3": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 14, "accuracy": 0.7777777777777778, "wilson95": [ 0.5478541568378739, 0.9099907189139831 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7777777777777778, "brier": 0.2740868133778357, "nll": 0.47898175165962176, "ece10": 0.2153616218886123, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.7777777777777777, "confusion_matrix": { "0": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 3, "2": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.14464593334187656, "nll": 0.3409362278748184, "ece10": 0.17328709788696606, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9532467532467532, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 4, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 3, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 13, "accuracy": 0.6190476190476191, "wilson95": [ 0.4087865377511576, 0.7924898863143551 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6190476190476191, "brier": 0.5111149660717265, "nll": 1.0051203762501593, "ece10": 0.30815884601940424, "calibration_n": 21, "confident_wrong": 0, "macro_f1": 0.6166946778711485, "confusion_matrix": { "spelling": { "spelling": 2, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 2, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 3, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 1, "word_choice": 1, "punctuation": 0, "none": 2, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 2, "none": 3, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.06665783932904537, "nll": 0.16153031282561492, "ece10": 0.14078122276852112, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.004303056055010652, "nll": 0.03327887957129268, "ece10": 0.03189475841313665, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 11, "accuracy": 0.7857142857142857, "wilson95": [ 0.524107694133997, 0.9242861328730683 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7857142857142857, "brier": 0.28916686995627344, "nll": 0.5496801479075638, "ece10": 0.1380234333811621, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 0.6992063492063492, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 1, "1": 4, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 14, "accuracy": 0.875, "wilson95": [ 0.639771727342413, 0.9650225122567595 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.2223936632925645, "nll": 0.4254956313760048, "ece10": 0.19451898424353237, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8875, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 13, "accuracy": 0.8125, "wilson95": [ 0.5699111903802586, 0.9340840092857187 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8125, "brier": 0.33850493423904493, "nll": 0.6662816898649937, "ece10": 0.16877211492254418, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.8031746031746032, "confusion_matrix": { "0": { "0": 2, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 13, "accuracy": 0.7647058823529411, "wilson95": [ 0.5273820188043501, 0.9044495567791988 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7647058823529411, "brier": 0.3968385337938196, "nll": 0.5878229540031831, "ece10": 0.2071213005666928, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.7424242424242424, "confusion_matrix": { "false": { "false": 4, "true": 4, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.036173328186023285, "nll": 0.08343801154111506, "ece10": 0.06652337047579009, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.05468751840887136, "nll": 0.14016294143363367, "ece10": 0.1144913559106123, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.0865500381015034, "nll": 0.19373994400279546, "ece10": 0.16433962440886968, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9411764705882353, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.022102306434178747, "nll": 0.08803975887108587, "ece10": 0.08170447153925216, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.285427734553092, "nll": 0.47533039657664256, "ece10": 0.15613519578796264, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.6825396825396824, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 4, "1": 1, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 16, "accuracy": 0.8888888888888888, "wilson95": [ 0.672002348698212, 0.9689804773543876 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8888888888888888, "brier": 0.10407351396715164, "nll": 0.19740043521341014, "ece10": 0.08989357796065867, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.8873015873015873, "confusion_matrix": { "italian": { "italian": 1, "french": 1, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 1, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 3, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 9, "accuracy": 0.5294117647058824, "wilson95": [ 0.30963235060004424, 0.7383489355759056 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 9, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5294117647058824, "brier": 0.48715124210793137, "nll": 0.6625106936273163, "ece10": 0.22304707823067618, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.48484848484848486, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 7, "true": 2, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0684727220397804, "nll": 0.15280662719417887, "ece10": 0.12622470107989797, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 4, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 11, "accuracy": 0.6111111111111112, "wilson95": [ 0.3861904160223791, 0.7969475342783638 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6111111111111112, "brier": 0.49240589843276006, "nll": 0.9418145931035878, "ece10": 0.13450643891835895, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.5880952380952381, "confusion_matrix": { "0": { "0": 1, "1": 2, "2": 0, "3": 1, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 1, "2": 2, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.19718715909959805, "nll": 0.32399470461355223, "ece10": 0.21041319353124735, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9352226720647774, "confusion_matrix": { "false": { "false": 9, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.22140150634443503, "nll": 0.3843518822311464, "ece10": 0.20592393456233501, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 0.9333333333333333, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.05998855514535461, "nll": 0.1388152194711132, "ece10": 0.10984787195686684, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9199999999999999, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 4, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 2, "other": 1, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.07986869743108926, "nll": 0.19499123854066608, "ece10": 0.16936083656565615, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 14, "accuracy": 0.7368421052631579, "wilson95": [ 0.5120844910218489, 0.8819358765321748 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7368421052631579, "brier": 0.31488637454624807, "nll": 0.5625920336257662, "ece10": 0.19272758712520763, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 0.5968406593406593, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 0, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 1, "2": 1, "3": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 12, "accuracy": 0.7058823529411765, "wilson95": [ 0.4686688993346016, 0.8672001038970478 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7058823529411765, "brier": 0.3248063989201593, "nll": 0.4832609300789966, "ece10": 0.12083633603266808, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.5502645502645502, "confusion_matrix": { "false": { "false": 11, "true": 0, "__failed__": 0 }, "true": { "false": 5, "true": 1, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.043323573033219626, "nll": 0.12654304218108808, "ece10": 0.11307721503936807, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 16, "accuracy": 0.8, "wilson95": [ 0.5839825677481064, 0.919342337420202 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.33226841150677056, "nll": 0.5786122374180684, "ece10": 0.15497438702330535, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.7776667776667777, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 19, "accuracy": 0.95, "wilson95": [ 0.763868806553258, 0.9911185511992047 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.038599521664136494, "nll": 0.06617872694677057, "ece10": 0.04691010780085099, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 0.9428571428571428, "confusion_matrix": { "device_control": { "device_control": 3, "information_request": 0, "reminder": 0, "music": 1, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 17, "accuracy": 0.85, "wilson95": [ 0.6395811352592431, 0.9476312541037835 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.2608756318714497, "nll": 0.430855259812496, "ece10": 0.13298409808810155, "calibration_n": 20, "confident_wrong": 1, "macro_f1": 0.849624060150376, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 3, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 20, "accuracy": 0.9090909090909091, "wilson95": [ 0.7218505315839417, 0.9747045710090046 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9090909090909091, "brier": 0.1635834490432141, "nll": 0.3796697507505425, "ece10": 0.22075569852805454, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 0.9142857142857143, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 2, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 595, "accuracy": 0.6611111111111111, "wilson95": [ 0.6295607631531338, 0.6912919673449599 ], "valid": 899, "failures": 1, "coverage": 0.9988888888888889, "heads": 1073, "head_correct": 747, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6961789375582479, "cluster_bootstrap95": [ 0.620132422925719, 0.7117335822534592 ], "independent_source_groups": 596, "brier": 0.4062691178794709, "nll": 0.7210373542183026, "ece10": 0.1091722957533334, "calibration_n": 1072, "confident_wrong": 35 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 403, "accuracy": 0.7020905923344948, "wilson95": [ 0.6634337654552191, 0.738060444519753 ], "valid": 566, "failures": 8, "coverage": 0.9860627177700348, "heads": 936, "head_correct": 749, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8002136752136753, "brier": 0.27592241841010406, "nll": 0.5324723322797632, "ece10": 0.03400024162609987, "calibration_n": 923, "confident_wrong": 26 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 363, "accuracy": 0.5185714285714286, "wilson95": [ 0.4815567977500565, 0.5553833393705441 ], "valid": 632, "failures": 68, "coverage": 0.9028571428571428, "heads": 1088, "head_correct": 649, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5965073529411765, "cluster_bootstrap95": [ 0.48200628229810516, 0.5547652916073968 ], "independent_source_groups": 650, "brier": 0.41178003668502616, "nll": 0.7502705782475058, "ece10": 0.043938605234018144, "calibration_n": 952, "confident_wrong": 8 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 86, "accuracy": 0.215, "wilson95": [ 0.17755148423753886, 0.25787052353301015 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1435, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7175, "brier": 0.3969547651688186, "nll": 0.6953854776854737, "ece10": 0.0731828673659745, "calibration_n": 2000, "confident_wrong": 2 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.015890638484713383, "nll": 0.07571043232168942, "ece10": 0.06987615015448606, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.027996442777577065, "nll": 0.09840553633209369, "ece10": 0.08777157615099701, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02213756744611981, "nll": 0.09849884313594634, "ece10": 0.09014334170938092, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03531926657030983, "nll": 0.12413698712639971, "ece10": 0.11065263237212966, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 36, "accuracy": 0.9, "wilson95": [ 0.7694822477247771, 0.9604204713173936 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 36, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.10514113222149957, "nll": 0.1982063566927094, "ece10": 0.07698771447127037, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8972672064777327, "confusion_matrix": { "review": { "review": 5, "it": 0, "documents": 1, "procurement": 0, "maintenance": 2, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 7, "procurement": 0, "maintenance": 1, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 38, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.95, "brier": 0.07359743294976957, "nll": 0.1643654173705141, "ece10": 0.0778437917864197, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.9492063492063492, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 6, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 35, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.875, "brier": 0.1810059939093987, "nll": 0.35058816629082235, "ece10": 0.09114130433442771, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 0.8552897395002658, "confusion_matrix": { "review": { "review": 3, "it": 0, "documents": 2, "procurement": 0, "maintenance": 3, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 36, "accuracy": 0.9, "wilson95": [ 0.7694822477247771, 0.9604204713173936 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 36, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.15776151887997925, "nll": 0.3406382326572287, "ece10": 0.06636204205656665, "calibration_n": 40, "confident_wrong": 1, "macro_f1": 0.8875816993464053, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 2, "it": 1, "review": 4, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 239, "accuracy": 0.5975, "wilson95": [ 0.5487342037489487, 0.6444108988558637 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 239, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5975, "brier": 0.5288215854601447, "nll": 1.0963693763905382, "ece10": 0.12019409609426185, "calibration_n": 400, "confident_wrong": 8, "macro_f1": 0.5055451211956482, "confusion_matrix": { "sadness": { "sadness": 95, "joy": 9, "love": 4, "anger": 4, "fear": 7, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 18, "joy": 70, "love": 17, "anger": 5, "fear": 7, "surprise": 1, "__failed__": 0 }, "love": { "sadness": 9, "joy": 12, "love": 9, "anger": 3, "fear": 3, "surprise": 1, "__failed__": 0 }, "anger": { "sadness": 22, "joy": 2, "love": 3, "anger": 33, "fear": 2, "surprise": 2, "__failed__": 0 }, "fear": { "sadness": 13, "joy": 1, "love": 1, "anger": 5, "fear": 29, "surprise": 3, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 1, "love": 1, "anger": 1, "fear": 2, "surprise": 3, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 276, "accuracy": 0.92, "wilson95": [ 0.8837267595665261, 0.9456531446048553 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 276, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.92, "brier": 0.11467202245079515, "nll": 0.26802143517013904, "ece10": 0.02573017701218902, "calibration_n": 300, "confident_wrong": 2 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 349, "accuracy": 0.8725, "wilson95": [ 0.836234529877535, 0.9016788108434143 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 349, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8725, "brier": 0.1846131709293624, "nll": 0.29252902336177866, "ece10": 0.032052320373299685, "calibration_n": 400, "confident_wrong": 3, "macro_f1": 0.8423188405797102, "confusion_matrix": { "false": { "false": 262, "true": 47, "__failed__": 0 }, "true": { "false": 4, "true": 87, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 338, "accuracy": 0.845, "wilson95": [ 0.8062691036268745, 0.8771674132824607 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 338, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.845, "brier": 0.23920570632284302, "nll": 0.4592251592229531, "ece10": 0.05057162308128304, "calibration_n": 400, "confident_wrong": 21, "macro_f1": 0.8304567316546951, "confusion_matrix": { "world": { "world": 88, "sports": 6, "business": 9, "sci_tech": 0, "__failed__": 0 }, "sports": { "world": 2, "sports": 121, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 3, "sports": 0, "business": 68, "sci_tech": 1, "__failed__": 0 }, "sci_tech": { "world": 10, "sports": 0, "business": 31, "sci_tech": 61, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 358, "accuracy": 0.895, "wilson95": [ 0.8611090149132419, 0.921376272562664 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 358, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.895, "brier": 0.14635101857437055, "nll": 0.24219358958733866, "ece10": 0.059972119725206344, "calibration_n": 400, "confident_wrong": 2, "macro_f1": 0.8825995807127882, "confusion_matrix": { "true": { "true": 114, "false": 39, "__failed__": 0 }, "false": { "true": 3, "false": 244, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 233, "accuracy": 0.7766666666666666, "wilson95": [ 0.7262089140024802, 0.8201286419199375 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 233, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7766666666666666, "brier": 0.34534226154045716, "nll": 0.798344941888022, "ece10": 0.11447559258918943, "calibration_n": 300, "confident_wrong": 14 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 363, "accuracy": 0.9075, "wilson95": [ 0.8751014146161221, 0.9321460655014261 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 363, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9075, "brier": 0.15161758637783385, "nll": 0.2579031654830427, "ece10": 0.04943300019769299, "calibration_n": 400, "confident_wrong": 1, "macro_f1": 0.9074299939329118, "confusion_matrix": { "false": { "false": 187, "true": 1, "__failed__": 0 }, "true": { "false": 36, "true": 176, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 278, "accuracy": 0.695, "wilson95": [ 0.6482028147696431, 0.7380873904399811 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 278, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.695, "brier": 0.3932911407814781, "nll": 0.5642658691159643, "ece10": 0.10295878350986458, "calibration_n": 400, "confident_wrong": 7, "macro_f1": 0.6728871728871728, "confusion_matrix": { "false": { "false": 191, "true": 9, "__failed__": 0 }, "true": { "false": 113, "true": 87, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 79, "accuracy": 0.79, "wilson95": [ 0.7002003116591013, 0.858343459380847 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 79, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.79, "brier": 0.30726679220729247, "nll": 0.4727872999907341, "ece10": 0.050481279554353095, "calibration_n": 100, "confident_wrong": 3, "macro_f1": 0.7851662404092072, "confusion_matrix": { "yes": { "yes": 47, "no": 7, "__failed__": 0 }, "no": { "yes": 14, "no": 32, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 50, "accuracy": 0.5, "wilson95": [ 0.4038315303659956, 0.5961684696340044 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 50, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.776814941087727, "nll": 1.983451264368454, "ece10": 0.3175310719362999, "calibration_n": 100, "confident_wrong": 19, "macro_f1": 0.5408158501715004, "confusion_matrix": { "balance": { "balance": 3, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 3, "dispute_charge": 1, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 2, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 3, "mortgage": 4, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 5, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 3, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 6, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 7, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 4, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 5, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 3, "limit_increase": 0, "other": 0, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 3, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 2, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 50, "accuracy": 0.5, "wilson95": [ 0.4038315303659956, 0.5961684696340044 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 146, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.73, "brier": 0.3640772790545473, "nll": 0.642720032669808, "ece10": 0.10968992122303438, "calibration_n": 200, "confident_wrong": 4 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 53, "accuracy": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 136, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.68, "brier": 0.49812198395113133, "nll": 1.106574843244696, "ece10": 0.12772806821943655, "calibration_n": 200, "confident_wrong": 17 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8, "brier": 0.3309876201972258, "nll": 0.8940021856969458, "ece10": 0.12481171134462221, "calibration_n": 100, "confident_wrong": 5, "macro_f1": 0.7450243036957035, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 13, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 12, "clinical_note": 0, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 8, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 6, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 5, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 203, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5075, "brier": 0.7256920752690579, "nll": 1.5439122686766893, "ece10": 0.24805908116074693, "calibration_n": 400, "confident_wrong": 51 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 63, "accuracy": 0.63, "wilson95": [ 0.5322052958094291, 0.7181763946567546 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 63, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.63, "brier": 0.5090791392842902, "nll": 1.2392041375957015, "ece10": 0.16422008042789063, "calibration_n": 100, "confident_wrong": 6, "macro_f1": 0.6122029562397041, "confusion_matrix": { "politics": { "politics": 0, "business": 0, "technology": 4, "science": 3, "health": 4, "sports": 0, "entertainment": 0, "world": 1, "culture": 1, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 8, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 2, "technology": 8, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 0, "business": 2, "technology": 1, "science": 6, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 6, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 0, "business": 0, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 3, "weather": 1, "__failed__": 0 }, "world": { "politics": 0, "business": 1, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 5, "culture": 1, "weather": 1, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 9, "weather": 0, "__failed__": 0 }, "weather": { "politics": 0, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 6, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 43, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43, "brier": 0.7218493861791573, "nll": 1.8378047601650964, "ece10": 0.17410744139027984, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.40732421372415467, "confusion_matrix": { "biology": { "biology": 5, "medicine": 0, "chemistry": 1, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 0, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 8, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 0, "medicine": 0, "chemistry": 4, "physics": 0, "computer_science": 1, "mathematics": 0, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 2, "medicine": 2, "chemistry": 0, "physics": 6, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 0, "medicine": 0, "chemistry": 2, "physics": 0, "computer_science": 5, "mathematics": 0, "earth_science": 2, "engineering": 6, "economics": 0, "other": 0, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 0, "chemistry": 1, "physics": 2, "computer_science": 2, "mathematics": 0, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 1, "computer_science": 1, "mathematics": 0, "earth_science": 6, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 1, "earth_science": 0, "engineering": 5, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 1, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 1, "economics": 4, "other": 0, "__failed__": 0 }, "other": { "biology": 0, "medicine": 0, "chemistry": 0, "physics": 2, "computer_science": 1, "mathematics": 2, "earth_science": 3, "engineering": 4, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 9, "accuracy": 0.09, "wilson95": [ 0.04807254000256516, 0.1622621285271631 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 80, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.4, "brier": 0.5031434128003176, "nll": 0.9932005288989203, "ece10": 0.1345921240809559, "calibration_n": 100, "confident_wrong": 7 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 20, "accuracy": 0.2, "wilson95": [ 0.1333669333310325, 0.2888291655931589 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 101, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.505, "brier": 0.35849506440978496, "nll": 0.628311483611331, "ece10": 0.11326474428283227, "calibration_n": 100, "confident_wrong": 0 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 78, "accuracy": 0.78, "wilson95": [ 0.6892964648501421, 0.8499871761539459 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 78, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.78, "brier": 0.2996605560728924, "nll": 0.5400842121174648, "ece10": 0.09392200502635491, "calibration_n": 100, "confident_wrong": 0, "macro_f1": 0.7647448885614293, "confusion_matrix": { "negative": { "negative": 36, "neutral": 0, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 15, "neutral": 15, "positive": 3, "__failed__": 0 }, "positive": { "negative": 2, "neutral": 2, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 93, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.465, "brier": 0.23840195669836317, "nll": 0.3713406589637584, "ece10": 0.06657892212179813, "calibration_n": 100, "confident_wrong": 4 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 40, "accuracy": 0.4, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 226, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7533333333333333, "brier": 0.36794241470612143, "nll": 0.7215245967936148, "ece10": 0.08920931656308666, "calibration_n": 300, "confident_wrong": 10 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 69, "accuracy": 0.69, "wilson95": [ 0.5937394428051057, 0.7722030278762395 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.69, "brier": 0.47203496697556785, "nll": 1.4729926686563968, "ece10": 0.10510191157538804, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.6539125244012463, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 1, "password_reset": 3, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 4, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 2, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 4, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 3, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 4, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 3, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 9, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 4, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 6, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 1, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 2, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 1, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 1, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 41, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.41, "brier": 0.9446615649659883, "nll": 2.477069074320959, "ece10": 0.35454626146764223, "calibration_n": 100, "confident_wrong": 19, "macro_f1": 0.40049437472298616, "confusion_matrix": { "billing": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 3, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 3, "account_access": 4, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 1, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 2, "returns": 1, "technical_outage": 1, "onboarding": 0, "integrations": 3, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 6, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 3, "onboarding": 0, "integrations": 2, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 1, "onboarding": 2, "integrations": 3, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 7, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "compliance": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 2, "technical_outage": 2, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 2, "pricing": 0, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 0, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 5, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 4, "other": 0, "__failed__": 0 }, "other": { "billing": 1, "account_access": 1, "shipping": 1, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 0, "compliance": 1, "pricing": 1, "data_privacy": 2, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 9, "accuracy": 0.09, "wilson95": [ 0.04807254000256516, 0.1622621285271631 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 145, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.48333333333333334, "brier": 0.6672605432154708, "nll": 1.5685212438798426, "ece10": 0.19569111753872862, "calibration_n": 300, "confident_wrong": 19 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 60, "accuracy": 0.6, "wilson95": [ 0.5020025867910618, 0.6905987135675411 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 60, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.687790369178783, "nll": 2.072687502305942, "ece10": 0.3217198842993352, "calibration_n": 100, "confident_wrong": 27, "macro_f1": 0.5671723930237863, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 4, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 1, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 2, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 8, "baggage": 0, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 12, "refund": 0, "hotel": 0, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 10, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 3, "loyalty": 1, "disruption": 0, "other": 0, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 1, "loyalty": 6, "disruption": 0, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 4, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 1, "refund": 0, "hotel": 4, "car": 2, "visa_docs": 1, "loyalty": 1, "disruption": 1, "other": 1, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 86, "accuracy": 0.86, "wilson95": [ 0.7786281178903616, 0.9147365634006086 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 86, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.86, "brier": 0.20145798026298622, "nll": 0.37251956216528015, "ece10": 0.06235691714783405, "calibration_n": 100, "confident_wrong": 1, "macro_f1": 0.8425925925925927, "confusion_matrix": { "business": { "business": 25, "science and technology": 0, "sports": 0, "world": 0, "__failed__": 0 }, "science and technology": { "business": 10, "science and technology": 11, "sports": 0, "world": 4, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 87, "accuracy": 0.87, "wilson95": [ 0.7901964855992357, 0.9224283257275948 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 87, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.87, "brier": 0.21326931799889345, "nll": 0.5125341327085452, "ece10": 0.028188376788929624, "calibration_n": 100, "confident_wrong": 2, "macro_f1": 0.8486111111111111, "confusion_matrix": { "refund not showing up": { "refund not showing up": 2, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "activating a card": { "refund not showing up": 0, "activating a card": 2, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 2, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 1, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 2, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 2, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 2, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 2, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 2, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 2, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 1, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 2, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 1, "card delivery estimate or timing": 1, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 2, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 2, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 1, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 2, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 2, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 2, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 2, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 2, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 2, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 2, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 1, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 1, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 1, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 1, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 2, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 2, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 1, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 1, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 1, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 1, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 1, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 1, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 1, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 1, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 1, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 1, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 1, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 1, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 1, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 1, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 1, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 1, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 1, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 1, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 1, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 1, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 1, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 1, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 1, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 1, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 1, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 1, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 1, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 1, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 1, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 1, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 1, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 1, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 1, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 1, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 1, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 1, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 1, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 1, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 1, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 1, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 0 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 1, "__failed__": 0 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 43, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.43, "brier": 0.7632722517408029, "nll": 1.5285524054704567, "ece10": 0.2805013047235382, "calibration_n": 100, "confident_wrong": 4, "macro_f1": 0.44871175523349444, "confusion_matrix": { "anger": { "anger": 8, "fear": 1, "joy": 0, "love": 0, "sadness": 8, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 1, "fear": 7, "joy": 0, "love": 0, "sadness": 8, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 0, "fear": 0, "joy": 8, "love": 2, "sadness": 5, "surprise": 2, "__failed__": 0 }, "love": { "anger": 0, "fear": 0, "joy": 3, "love": 5, "sadness": 9, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 2, "joy": 2, "love": 1, "sadness": 11, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 0, "joy": 2, "love": 0, "sadness": 9, "surprise": 4, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2507, "accuracy": 0.8504070556309362, "wilson95": [ 0.8370761654409344, 0.8628259223986334 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2507, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8504070556309362, "brier": 0.22324343600280155, "nll": 0.526456339696134, "ece10": 0.047728502784061363, "calibration_n": 2948, "confident_wrong": 97, "macro_f1": 0.8531678951755448, "confusion_matrix": { "alarm": { "alarm": 91, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 51, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 1, "audio": 0, "calendar": 370, "cooking": 0, "datetime": 4, "email": 0, "general": 11, "iot": 0, "lists": 2, "music": 0, "news": 3, "play": 0, "qa": 3, "recommendation": 4, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 68, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 93, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 6, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 0, "email": 229, "general": 27, "iot": 0, "lists": 6, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 10, "cooking": 4, "datetime": 0, "email": 0, "general": 117, "iot": 2, "lists": 0, "music": 1, "news": 4, "play": 2, "qa": 30, "recommendation": 9, "social": 2, "takeaway": 0, "transport": 3, "weather": 2, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 2, "calendar": 0, "cooking": 2, "datetime": 1, "email": 0, "general": 10, "iot": 198, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 7, "cooking": 0, "datetime": 0, "email": 0, "general": 10, "iot": 1, "lists": 120, "music": 3, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 8, "iot": 0, "lists": 0, "music": 64, "news": 0, "play": 3, "qa": 4, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 0, "lists": 0, "music": 0, "news": 109, "play": 0, "qa": 8, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 24, "iot": 9, "lists": 0, "music": 49, "news": 1, "play": 293, "qa": 6, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 24, "iot": 0, "lists": 1, "music": 1, "news": 4, "play": 0, "qa": 251, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 2, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 6, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 0, "lists": 0, "music": 0, "news": 7, "play": 0, "qa": 6, "recommendation": 68, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 4, "general": 14, "iot": 0, "lists": 0, "music": 0, "news": 3, "play": 0, "qa": 1, "recommendation": 0, "social": 83, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 3, "datetime": 0, "email": 0, "general": 2, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 4, "recommendation": 1, "social": 0, "takeaway": 44, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 115, "weather": 2, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 143, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2379, "accuracy": 0.8069877883310719, "wilson95": [ 0.7923453834065616, 0.8208311795119368 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2379, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8069877883310719, "brier": 0.2755579649278625, "nll": 0.6504083407363339, "ece10": 0.0549482375789279, "calibration_n": 2948, "confident_wrong": 91, "macro_f1": 0.8060115914062506, "confusion_matrix": { "alarm": { "alarm": 83, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 2, "email": 0, "general": 1, "iot": 6, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 2, "audio": 45, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 9, "iot": 0, "lists": 0, "music": 1, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 2, "audio": 0, "calendar": 354, "cooking": 1, "datetime": 6, "email": 1, "general": 15, "iot": 1, "lists": 2, "music": 0, "news": 4, "play": 0, "qa": 4, "recommendation": 7, "social": 1, "takeaway": 0, "transport": 1, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 66, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 96, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 4, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 0, "datetime": 0, "email": 229, "general": 22, "iot": 1, "lists": 3, "music": 0, "news": 0, "play": 0, "qa": 8, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 1, "calendar": 9, "cooking": 5, "datetime": 0, "email": 2, "general": 124, "iot": 2, "lists": 0, "music": 1, "news": 7, "play": 1, "qa": 24, "recommendation": 5, "social": 1, "takeaway": 0, "transport": 3, "weather": 2, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 4, "calendar": 0, "cooking": 10, "datetime": 1, "email": 0, "general": 13, "iot": 182, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 2, "calendar": 4, "cooking": 0, "datetime": 0, "email": 0, "general": 11, "iot": 4, "lists": 117, "music": 2, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 8, "iot": 1, "lists": 1, "music": 63, "news": 0, "play": 3, "qa": 3, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 12, "iot": 0, "lists": 1, "music": 0, "news": 99, "play": 0, "qa": 9, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "play": { "alarm": 0, "audio": 11, "calendar": 0, "cooking": 1, "datetime": 2, "email": 0, "general": 37, "iot": 7, "lists": 3, "music": 61, "news": 0, "play": 256, "qa": 4, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 37, "iot": 0, "lists": 0, "music": 1, "news": 8, "play": 0, "qa": 234, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 2, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 9, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 0, "lists": 0, "music": 0, "news": 5, "play": 0, "qa": 7, "recommendation": 64, "social": 0, "takeaway": 0, "transport": 2, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 4, "general": 21, "iot": 0, "lists": 0, "music": 0, "news": 2, "play": 0, "qa": 1, "recommendation": 0, "social": 77, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 3, "datetime": 0, "email": 0, "general": 12, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 2, "social": 0, "takeaway": 37, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 1, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 112, "weather": 3, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 141, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 854, "accuracy": 0.7280477408354646, "wilson95": [ 0.7018701174242261, 0.7527365723639807 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 854, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7280477408354646, "cluster_bootstrap95": [ 0.6950177453250986, 0.7620136716023224 ], "independent_source_groups": 47, "brier": 0.39483958530034335, "nll": 0.6880391930049211, "ece10": 0.06725795653536142, "calibration_n": 1173, "confident_wrong": 4, "macro_f1": 0.6738302055423057, "confusion_matrix": { "negative": { "negative": 87, "neutral": 82, "positive": 13, "__failed__": 0 }, "neutral": { "negative": 46, "neutral": 523, "positive": 21, "__failed__": 0 }, "positive": { "negative": 22, "neutral": 135, "positive": 244, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 }, "winnow-12b-q8": { "label": "Winnow 12B Q8 (Windows CUDA)", "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 170, "accuracy": 0.85, "wilson95": [ 0.7939442071583334, 0.89286406437758 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 170, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.22991373488346414, "nll": 0.48387572336430573, "ece10": 0.10875179318730327, "calibration_n": 200, "confident_wrong": 16, "macro_f1": 0.8540825561713159, "confusion_matrix": { "world": { "world": 43, "sports": 0, "business": 4, "sci_tech": 3, "__failed__": 0 }, "sports": { "world": 0, "sports": 45, "business": 2, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 3, "sports": 0, "business": 46, "sci_tech": 2, "__failed__": 0 }, "sci_tech": { "world": 2, "sports": 0, "business": 14, "sci_tech": 36, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.26, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 120, "accuracy": 0.6, "wilson95": [ 0.53083672039262, 0.6653942143319266 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 120, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6, "brier": 0.7099915800277757, "nll": 1.932629474387965, "ece10": 0.33175273397462285, "calibration_n": 200, "confident_wrong": 56, "macro_f1": 0.49685585397223647, "confusion_matrix": { "sadness": { "sadness": 44, "joy": 2, "love": 2, "anger": 4, "fear": 5, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 17, "joy": 47, "love": 9, "anger": 3, "fear": 3, "surprise": 0, "__failed__": 0 }, "love": { "sadness": 2, "joy": 5, "love": 4, "anger": 1, "fear": 0, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 7, "joy": 3, "love": 1, "anger": 15, "fear": 2, "surprise": 0, "__failed__": 0 }, "fear": { "sadness": 6, "joy": 3, "love": 0, "anger": 1, "fear": 9, "surprise": 1, "__failed__": 0 }, "surprise": { "sadness": 0, "joy": 3, "love": 0, "anger": 0, "fear": 0, "surprise": 1, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.395, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 187, "accuracy": 0.935, "wilson95": [ 0.8919809207009312, 0.961623645350847 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 187, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.935, "brier": 0.10137991519720752, "nll": 0.17250539217012834, "ece10": 0.032006804089195026, "calibration_n": 200, "confident_wrong": 5, "macro_f1": 0.9337933844312597, "confusion_matrix": { "false": { "false": 107, "true": 2, "__failed__": 0 }, "true": { "false": 11, "true": 80, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.545, "selective_policy": { "threshold": 0.98, "development": { "n": 73, "correct": 73, "wilson95": [ 0.9500079920442688, 0.9999999999999998 ] }, "test_coverage": 0.7, "accepted_n": 140, "accepted_correct": 139, "accepted_accuracy": 0.9928571428571429, "accepted_wilson95": [ 0.9606516069372742, 0.9987379955557328 ], "errors_per_1000_all_test_cases": 5.0, "note": "Selected on development only, applied unchanged to test. An empirical pilot estimate, not a production guarantee." } }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "n": 200, "correct": 182, "accuracy": 0.91, "wilson95": [ 0.8622342812328346, 0.9423125511378069 ], "valid": 200, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 182, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.91, "brier": 0.16216126970174957, "nll": 0.3133796680086965, "ece10": 0.06153899646148562, "calibration_n": 200, "confident_wrong": 11, "macro_f1": 0.9017359973796266, "confusion_matrix": { "true": { "true": 62, "false": 15, "__failed__": 0 }, "false": { "true": 3, "false": 120, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.615, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.16072126645934334, "nll": 0.3639925561663713, "ece10": 0.06980280294806339, "calibration_n": 12, "confident_wrong": 1, "macro_f1": 0.9314285714285713, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 1, "it": 0, "review": 3, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 11, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9166666666666666, "brier": 0.07753126305423035, "nll": 0.11523217686696946, "ece10": 0.06315866671721578, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 0.9314285714285713, "confusion_matrix": { "maintenance": { "maintenance": 2, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 2, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 2, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 2, "review": 0, "__failed__": 0 }, "review": { "maintenance": 1, "procurement": 0, "documents": 0, "it": 0, "review": 3, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.3333333333333333, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 7.950720075227105e-06, "nll": 0.0016151807234386698, "ece10": 0.001612598758534367, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "n": 12, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "valid": 12, "failures": 0, "coverage": 1.0, "heads": 12, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 7.55380068824807e-05, "nll": 0.0035284752763928904, "ece10": 0.0035070265973596673, "calibration_n": 12, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 3, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 3, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 3, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 3, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25, "selective_policy": { "threshold": null, "test_coverage": 0.0, "accepted_n": 0, "accepted_accuracy": null, "reason": "No predeclared threshold passed a 95% development Wilson lower bound with at least 30 accepted examples. All cases remain for review." } }, "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 88, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.88, "brier": 0.1980908146656793, "nll": 0.3973559819715017, "ece10": 0.07743457587796393, "calibration_n": 100, "confident_wrong": 7, "macro_f1": 0.8675169687777176, "confusion_matrix": { "label_000": { "label_000": 25, "label_001": 0, "label_002": 0, "label_003": 0, "__failed__": 0 }, "label_001": { "label_000": 8, "label_001": 13, "label_002": 1, "label_003": 3, "__failed__": 0 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 25, "label_003": 0, "__failed__": 0 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 25, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003" ], "retrospective_majority_baseline": 0.25 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 52, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.52, "brier": 0.8032170193394372, "nll": 2.288914917271393, "ece10": 0.33328254780708955, "calibration_n": 100, "confident_wrong": 23, "macro_f1": 0.5101097065702608, "confusion_matrix": { "label_000": { "label_000": 10, "label_001": 3, "label_002": 0, "label_003": 1, "label_004": 3, "label_005": 0, "__failed__": 0 }, "label_001": { "label_000": 2, "label_001": 11, "label_002": 0, "label_003": 0, "label_004": 3, "label_005": 1, "__failed__": 0 }, "label_002": { "label_000": 1, "label_001": 0, "label_002": 11, "label_003": 2, "label_004": 2, "label_005": 1, "__failed__": 0 }, "label_003": { "label_000": 1, "label_001": 1, "label_002": 2, "label_003": 7, "label_004": 6, "label_005": 0, "__failed__": 0 }, "label_004": { "label_000": 0, "label_001": 2, "label_002": 2, "label_003": 2, "label_004": 10, "label_005": 0, "__failed__": 0 }, "label_005": { "label_000": 1, "label_001": 0, "label_002": 5, "label_003": 0, "label_004": 7, "label_005": 3, "__failed__": 0 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005" ], "retrospective_majority_baseline": 0.17 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "label_000": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_001": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_002": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_003": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_004": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_005": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_006": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_007": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_008": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_009": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_010": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_011": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_012": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_013": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_014": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_015": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_016": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_017": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_018": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_019": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_020": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_021": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_022": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_023": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_024": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_025": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_026": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_027": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 2 }, "label_028": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_029": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_030": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_031": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_032": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_033": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_034": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_035": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_036": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_037": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_038": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_039": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_040": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_041": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_042": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_043": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_044": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_045": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_046": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_047": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_048": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_049": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_050": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_051": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_052": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_053": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_054": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_055": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_056": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_057": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_058": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_059": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_060": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_061": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_062": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_063": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_064": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_065": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_066": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_067": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_068": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_069": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_070": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 }, "label_071": { "label_000": 0, "label_001": 0, "label_002": 0, "label_003": 0, "label_004": 0, "label_005": 0, "label_006": 0, "label_007": 0, "label_008": 0, "label_009": 0, "label_010": 0, "label_011": 0, "label_012": 0, "label_013": 0, "label_014": 0, "label_015": 0, "label_016": 0, "label_017": 0, "label_018": 0, "label_019": 0, "label_020": 0, "label_021": 0, "label_022": 0, "label_023": 0, "label_024": 0, "label_025": 0, "label_026": 0, "label_027": 0, "label_028": 0, "label_029": 0, "label_030": 0, "label_031": 0, "label_032": 0, "label_033": 0, "label_034": 0, "label_035": 0, "label_036": 0, "label_037": 0, "label_038": 0, "label_039": 0, "label_040": 0, "label_041": 0, "label_042": 0, "label_043": 0, "label_044": 0, "label_045": 0, "label_046": 0, "label_047": 0, "label_048": 0, "label_049": 0, "label_050": 0, "label_051": 0, "label_052": 0, "label_053": 0, "label_054": 0, "label_055": 0, "label_056": 0, "label_057": 0, "label_058": 0, "label_059": 0, "label_060": 0, "label_061": 0, "label_062": 0, "label_063": 0, "label_064": 0, "label_065": 0, "label_066": 0, "label_067": 0, "label_068": 0, "label_069": 0, "label_070": 0, "label_071": 0, "__failed__": 1 } }, "labels": [ "label_000", "label_001", "label_002", "label_003", "label_004", "label_005", "label_006", "label_007", "label_008", "label_009", "label_010", "label_011", "label_012", "label_013", "label_014", "label_015", "label_016", "label_017", "label_018", "label_019", "label_020", "label_021", "label_022", "label_023", "label_024", "label_025", "label_026", "label_027", "label_028", "label_029", "label_030", "label_031", "label_032", "label_033", "label_034", "label_035", "label_036", "label_037", "label_038", "label_039", "label_040", "label_041", "label_042", "label_043", "label_044", "label_045", "label_046", "label_047", "label_048", "label_049", "label_050", "label_051", "label_052", "label_053", "label_054", "label_055", "label_056", "label_057", "label_058", "label_059", "label_060", "label_061", "label_062", "label_063", "label_064", "label_065", "label_066", "label_067", "label_068", "label_069", "label_070", "label_071" ], "retrospective_majority_baseline": 0.02 }, "jevbench_public/easy": { "expected": 48, "attempted": 48, "complete": true, "n": 48, "correct": 48, "accuracy": 1.0, "wilson95": [ 0.9258998703338824, 0.9999999999999999 ], "valid": 48, "failures": 0, "coverage": 1.0, "heads": 48, "head_correct": 48, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 6.311851820113444e-07, "nll": 0.0003051171200529249, "ece10": 0.0003049503402669009, "calibration_n": 48, "confident_wrong": 0 }, "jevbench_public/hard": { "expected": 111, "attempted": 111, "complete": true, "n": 111, "correct": 82, "accuracy": 0.7387387387387387, "wilson95": [ 0.6500078016644905, 0.8114980042026286 ], "valid": 111, "failures": 0, "coverage": 1.0, "heads": 111, "head_correct": 82, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7387387387387387, "brier": 0.3853739169168165, "nll": 0.71212150675567, "ece10": 0.12161530846843663, "calibration_n": 111, "confident_wrong": 8 }, "jevbench_public/original": { "expected": 72, "attempted": 72, "complete": true, "n": 72, "correct": 69, "accuracy": 0.9583333333333334, "wilson95": [ 0.8845072325868389, 0.9857291925330748 ], "valid": 72, "failures": 0, "coverage": 1.0, "heads": 72, "head_correct": 69, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9583333333333334, "cluster_bootstrap95": [ 0.8888888888888888, 1.0 ], "independent_source_groups": 36, "brier": 0.060595665247630014, "nll": 0.11040743962342657, "ece10": 0.026444791368185665, "calibration_n": 72, "confident_wrong": 1 }, "jabr_v2/action_item_assignment": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 17, "accuracy": 0.8947368421052632, "wilson95": [ 0.6860591728889099, 0.9706414397011299 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8947368421052632, "brier": 0.19773352083518284, "nll": 0.3993744683671187, "ece10": 0.09383714398664353, "calibration_n": 19, "confident_wrong": 2, "macro_f1": 0.8944444444444444, "confusion_matrix": { "false": { "false": 9, "true": 2, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5789473684210527 }, "jabr_v2/ad_policy_violation": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.003669426864900977, "nll": 0.02358239286294904, "ece10": 0.02257184532949072, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/allergen_present": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 22, "accuracy": 1.0, "wilson95": [ 0.8513451253913791, 1.0 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0016727320133366813, "nll": 0.02104458803794934, "ece10": 0.02061083334390457, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 10, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 12, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5454545454545454 }, "jabr_v2/app_review_intent": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00010559198423805167, "nll": 0.0034101034531079017, "ece10": 0.003382145701380644, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "bug_report": { "bug_report": 5, "feature_request": 0, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "feature_request": { "bug_report": 0, "feature_request": 4, "praise": 0, "pricing_billing": 0, "__failed__": 0 }, "praise": { "bug_report": 0, "feature_request": 0, "praise": 3, "pricing_billing": 0, "__failed__": 0 }, "pricing_billing": { "bug_report": 0, "feature_request": 0, "praise": 0, "pricing_billing": 4, "__failed__": 0 } }, "labels": [ "bug_report", "feature_request", "praise", "pricing_billing" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/churn_risk": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.000592537979889575, "nll": 0.013549437573165695, "ece10": 0.013360835158619806, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/city_service_request": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0009631897820442875, "nll": 0.007365923174492598, "ece10": 0.007096991891007408, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "sanitation": { "sanitation": 4, "roads": 0, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "roads": { "sanitation": 0, "roads": 4, "noise": 0, "permits": 0, "utilities": 0, "__failed__": 0 }, "noise": { "sanitation": 0, "roads": 0, "noise": 4, "permits": 0, "utilities": 0, "__failed__": 0 }, "permits": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 4, "utilities": 0, "__failed__": 0 }, "utilities": { "sanitation": 0, "roads": 0, "noise": 0, "permits": 0, "utilities": 3, "__failed__": 0 } }, "labels": [ "sanitation", "roads", "noise", "permits", "utilities" ], "retrospective_majority_baseline": 0.21052631578947367 }, "jabr_v2/code_review_intent": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 14, "accuracy": 0.8235294117647058, "wilson95": [ 0.5897054143116588, 0.9380887336237901 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8235294117647058, "brier": 0.18331859415939963, "nll": 0.2850852882356118, "ece10": 0.132487513441396, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.8111111111111112, "confusion_matrix": { "bug": { "bug": 3, "performance": 0, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "performance": { "bug": 0, "performance": 2, "security": 0, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "security": { "bug": 0, "performance": 0, "security": 4, "style": 0, "refactor": 0, "nit": 0, "__failed__": 0 }, "style": { "bug": 0, "performance": 0, "security": 0, "style": 2, "refactor": 1, "nit": 0, "__failed__": 0 }, "refactor": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 1, "nit": 0, "__failed__": 0 }, "nit": { "bug": 0, "performance": 0, "security": 0, "style": 0, "refactor": 2, "nit": 2, "__failed__": 0 } }, "labels": [ "bug", "performance", "security", "style", "refactor", "nit" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/commit_intent": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 14, "accuracy": 0.9333333333333333, "wilson95": [ 0.7018347012621998, 0.9881331045067315 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9333333333333333, "brier": 0.11715614968613898, "nll": 0.1978307260342463, "ece10": 0.0562994920035253, "calibration_n": 15, "confident_wrong": 1, "macro_f1": 0.9259259259259259, "confusion_matrix": { "feature": { "feature": 3, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "fix": { "feature": 0, "fix": 2, "refactor": 0, "docs": 0, "test": 0, "chore": 0, "__failed__": 0 }, "refactor": { "feature": 0, "fix": 0, "refactor": 1, "docs": 0, "test": 0, "chore": 1, "__failed__": 0 }, "docs": { "feature": 0, "fix": 0, "refactor": 0, "docs": 2, "test": 0, "chore": 0, "__failed__": 0 }, "test": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 2, "chore": 0, "__failed__": 0 }, "chore": { "feature": 0, "fix": 0, "refactor": 0, "docs": 0, "test": 0, "chore": 4, "__failed__": 0 } }, "labels": [ "feature", "fix", "refactor", "docs", "test", "chore" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/contains_pii": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.10494596921807108, "nll": 0.1788129999934973, "ece10": 0.04472110473174445, "calibration_n": 17, "confident_wrong": 1, "macro_f1": 0.9403508771929825, "confusion_matrix": { "false": { "false": 7, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/contains_spoiler": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.004556450899128467, "nll": 0.027818039296236243, "ece10": 0.026576275289406003, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/content_moderation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.03036628537477222, "nll": 0.06186981168673059, "ece10": 0.05088633667377332, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "harassment": { "harassment": 5, "threat": 0, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "threat": { "harassment": 0, "threat": 2, "self_harm": 0, "spam": 0, "none": 0, "__failed__": 0 }, "self_harm": { "harassment": 0, "threat": 0, "self_harm": 1, "spam": 0, "none": 0, "__failed__": 0 }, "spam": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 4, "none": 0, "__failed__": 0 }, "none": { "harassment": 0, "threat": 0, "self_harm": 0, "spam": 0, "none": 6, "__failed__": 0 } }, "labels": [ "harassment", "threat", "self_harm", "spam", "none" ], "retrospective_majority_baseline": 0.3333333333333333 }, "jabr_v2/content_type": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 21, "accuracy": 0.9545454545454546, "wilson95": [ 0.7820195416134267, 0.991930572378736 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 21, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9545454545454546, "brier": 0.08739654334877336, "nll": 0.15945644606201598, "ece10": 0.05200240712048558, "calibration_n": 22, "confident_wrong": 1, "macro_f1": 0.9532467532467532, "confusion_matrix": { "news_report": { "news_report": 5, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "opinion": { "news_report": 0, "opinion": 5, "press_release": 0, "analysis": 0, "feature": 0, "__failed__": 0 }, "press_release": { "news_report": 0, "opinion": 0, "press_release": 4, "analysis": 0, "feature": 0, "__failed__": 0 }, "analysis": { "news_report": 0, "opinion": 0, "press_release": 0, "analysis": 4, "feature": 0, "__failed__": 0 }, "feature": { "news_report": 1, "opinion": 0, "press_release": 0, "analysis": 0, "feature": 3, "__failed__": 0 } }, "labels": [ "news_report", "opinion", "press_release", "analysis", "feature" ], "retrospective_majority_baseline": 0.22727272727272727 }, "jabr_v2/contract_clause_type": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 2.5235670263685017e-05, "nll": 0.0017533260941664502, "ece10": 0.0017461035617042898, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "limitation_of_liability": { "limitation_of_liability": 3, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "confidentiality": { "limitation_of_liability": 0, "confidentiality": 4, "termination": 0, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "termination": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 3, "indemnification": 0, "force_majeure": 0, "__failed__": 0 }, "indemnification": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 4, "force_majeure": 0, "__failed__": 0 }, "force_majeure": { "limitation_of_liability": 0, "confidentiality": 0, "termination": 0, "indemnification": 0, "force_majeure": 3, "__failed__": 0 } }, "labels": [ "limitation_of_liability", "confidentiality", "termination", "indemnification", "force_majeure" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/delivery_exception": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 19, "accuracy": 1.0, "wilson95": [ 0.8318207759473837, 1.0 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 19, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 8.43190787835129e-05, "nll": 0.003931902543984639, "ece10": 0.0039039838934845053, "calibration_n": 19, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "address_issue": { "address_issue": 4, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "customs_hold": { "address_issue": 0, "customs_hold": 5, "weather_delay": 0, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "weather_delay": { "address_issue": 0, "customs_hold": 0, "weather_delay": 3, "failed_attempt": 0, "damaged": 0, "__failed__": 0 }, "failed_attempt": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 3, "damaged": 0, "__failed__": 0 }, "damaged": { "address_issue": 0, "customs_hold": 0, "weather_delay": 0, "failed_attempt": 0, "damaged": 4, "__failed__": 0 } }, "labels": [ "address_issue", "customs_hold", "weather_delay", "failed_attempt", "damaged" ], "retrospective_majority_baseline": 0.2631578947368421 }, "jabr_v2/dietary_vegan": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0001862545391417131, "nll": 0.007607598823818755, "ece10": 0.00756057164786211, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/document_type": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 3.9247935194035526e-05, "nll": 0.001965553282090236, "ece10": 0.00195402725793233, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "invoice": { "invoice": 3, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "purchase_order": { "invoice": 0, "purchase_order": 2, "receipt": 0, "contract": 0, "resume": 0, "__failed__": 0 }, "receipt": { "invoice": 0, "purchase_order": 0, "receipt": 3, "contract": 0, "resume": 0, "__failed__": 0 }, "contract": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 4, "resume": 0, "__failed__": 0 }, "resume": { "invoice": 0, "purchase_order": 0, "receipt": 0, "contract": 0, "resume": 3, "__failed__": 0 } }, "labels": [ "invoice", "purchase_order", "receipt", "contract", "resume" ], "retrospective_majority_baseline": 0.26666666666666666 }, "jabr_v2/email_intent_v2": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0026179970469444123, "nll": 0.017010915521128076, "ece10": 0.016283697985117036, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "meeting_request": { "meeting_request": 3, "question": 0, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "question": { "meeting_request": 0, "question": 3, "complaint": 0, "thanks": 0, "fyi": 0, "__failed__": 0 }, "complaint": { "meeting_request": 0, "question": 0, "complaint": 3, "thanks": 0, "fyi": 0, "__failed__": 0 }, "thanks": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 3, "fyi": 0, "__failed__": 0 }, "fyi": { "meeting_request": 0, "question": 0, "complaint": 0, "thanks": 0, "fyi": 3, "__failed__": 0 } }, "labels": [ "meeting_request", "question", "complaint", "thanks", "fyi" ], "retrospective_majority_baseline": 0.2 }, "jabr_v2/expense_category": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.008676531257760333, "nll": 0.021518808551574044, "ece10": 0.018804600835231517, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "meals": { "meals": 2, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "travel": { "meals": 0, "travel": 5, "office_supplies": 0, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "office_supplies": { "meals": 0, "travel": 0, "office_supplies": 4, "software": 0, "client_entertainment": 0, "__failed__": 0 }, "software": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 4, "client_entertainment": 0, "__failed__": 0 }, "client_entertainment": { "meals": 0, "travel": 0, "office_supplies": 0, "software": 0, "client_entertainment": 3, "__failed__": 0 } }, "labels": [ "meals", "travel", "office_supplies", "software", "client_entertainment" ], "retrospective_majority_baseline": 0.2777777777777778 }, "jabr_v2/fair_housing_violation": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.01239970787450269, "nll": 0.04037055799729753, "ece10": 0.03656753694509335, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/formality_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.06589191784136354, "nll": 0.11140927794879427, "ece10": 0.07757612044909197, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9415584415584416, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 3, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/frustration_level_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.003731612794922758, "nll": 0.03368141844646547, "ece10": 0.03265618426896253, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 7, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.3888888888888889 }, "jabr_v2/gaming_report_type": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 18, "accuracy": 0.9, "wilson95": [ 0.6989663547715128, 0.9721335187862319 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.18141257259043667, "nll": 0.33398939079529055, "ece10": 0.09377701889700576, "calibration_n": 20, "confident_wrong": 2, "macro_f1": 0.891024531024531, "confusion_matrix": { "cheating": { "cheating": 4, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "harassment": { "cheating": 0, "harassment": 4, "boosting": 0, "smurfing": 0, "other": 0, "__failed__": 0 }, "boosting": { "cheating": 0, "harassment": 0, "boosting": 3, "smurfing": 0, "other": 1, "__failed__": 0 }, "smurfing": { "cheating": 1, "harassment": 0, "boosting": 0, "smurfing": 2, "other": 0, "__failed__": 0 }, "other": { "cheating": 0, "harassment": 0, "boosting": 0, "smurfing": 0, "other": 5, "__failed__": 0 } }, "labels": [ "cheating", "harassment", "boosting", "smurfing", "other" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/grammar_issue": { "expected": 21, "attempted": 21, "complete": true, "n": 21, "correct": 20, "accuracy": 0.9523809523809523, "wilson95": [ 0.7733063992587743, 0.9915440121901737 ], "valid": 21, "failures": 0, "coverage": 1.0, "heads": 21, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9523809523809523, "brier": 0.10704679655333871, "nll": 0.18815679210269182, "ece10": 0.07087784093012192, "calibration_n": 21, "confident_wrong": 1, "macro_f1": 0.9595959595959596, "confusion_matrix": { "spelling": { "spelling": 4, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "subject_verb_agreement": { "spelling": 0, "subject_verb_agreement": 3, "word_choice": 0, "punctuation": 0, "none": 0, "__failed__": 0 }, "word_choice": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 4, "punctuation": 0, "none": 0, "__failed__": 0 }, "punctuation": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 4, "none": 1, "__failed__": 0 }, "none": { "spelling": 0, "subject_verb_agreement": 0, "word_choice": 0, "punctuation": 0, "none": 5, "__failed__": 0 } }, "labels": [ "spelling", "subject_verb_agreement", "word_choice", "punctuation", "none" ], "retrospective_majority_baseline": 0.23809523809523808 }, "jabr_v2/hazmat_shipping": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.011785189203653715, "nll": 0.030389083433908795, "ece10": 0.026681087774003166, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 7, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5625 }, "jabr_v2/home_service_routing": { "expected": 23, "attempted": 23, "complete": true, "n": 23, "correct": 23, "accuracy": 1.0, "wilson95": [ 0.8568833815495731, 1.0 ], "valid": 23, "failures": 0, "coverage": 1.0, "heads": 23, "head_correct": 23, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 9.138481990756441e-06, "nll": 0.001084608535194724, "ece10": 0.001081524679688569, "calibration_n": 23, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "plumbing": { "plumbing": 5, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "electrical": { "plumbing": 0, "electrical": 5, "hvac": 0, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "hvac": { "plumbing": 0, "electrical": 0, "hvac": 4, "handyman": 0, "appliance_repair": 0, "__failed__": 0 }, "handyman": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 5, "appliance_repair": 0, "__failed__": 0 }, "appliance_repair": { "plumbing": 0, "electrical": 0, "hvac": 0, "handyman": 0, "appliance_repair": 4, "__failed__": 0 } }, "labels": [ "plumbing", "electrical", "hvac", "handyman", "appliance_repair" ], "retrospective_majority_baseline": 0.21739130434782608 }, "jabr_v2/incident_severity_v2": { "expected": 14, "attempted": 14, "complete": true, "n": 14, "correct": 14, "accuracy": 1.0, "wilson95": [ 0.7846891972623643, 1.0 ], "valid": 14, "failures": 0, "coverage": 1.0, "heads": 14, "head_correct": 14, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.003421346538266528, "nll": 0.03243241220006019, "ece10": 0.03132997061948318, "calibration_n": 14, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 3, "1": 0, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 2, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.35714285714285715 }, "jabr_v2/insurance_claim_priority": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.3423901925907154, "nll": 0.5041098314466439, "ece10": 0.22517961128845204, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7472222222222222, "confusion_matrix": { "0": { "0": 4, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 2, "1": 2, "2": 0, "3": 1, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3125 }, "jabr_v2/lead_qualification": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 12, "accuracy": 0.75, "wilson95": [ 0.5050168346449122, 0.8981793250878696 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 12, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.75, "brier": 0.29915207995070864, "nll": 0.4841374955772574, "ece10": 0.177577921357722, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.7183261183261183, "confusion_matrix": { "0": { "0": 3, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 1, "3": 1, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/meeting_conflict": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.020518488893970444, "nll": 0.04630285458527035, "ece10": 0.039199800868110125, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/news_topic": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0001763334328155783, "nll": 0.005770243641228289, "ece10": 0.0057116999619143145, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "business": { "politics": 0, "business": 4, "technology": 0, "sports": 0, "culture": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 0, "technology": 3, "sports": 0, "culture": 0, "__failed__": 0 }, "sports": { "politics": 0, "business": 0, "technology": 0, "sports": 3, "culture": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "sports": 0, "culture": 3, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "sports", "culture" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/oncall_route": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0011462752484968613, "nll": 0.013744190341686578, "ece10": 0.013403334104407501, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "payments": { "payments": 4, "web": 0, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "web": { "payments": 0, "web": 4, "infra": 0, "data": 0, "model": 0, "__failed__": 0 }, "infra": { "payments": 0, "web": 0, "infra": 4, "data": 0, "model": 0, "__failed__": 0 }, "data": { "payments": 0, "web": 0, "infra": 0, "data": 3, "model": 0, "__failed__": 0 }, "model": { "payments": 0, "web": 0, "infra": 0, "data": 0, "model": 2, "__failed__": 0 } }, "labels": [ "payments", "web", "infra", "data", "model" ], "retrospective_majority_baseline": 0.23529411764705882 }, "jabr_v2/phishing_email": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.003968121943660773, "nll": 0.02376216898882061, "ece10": 0.022663800321437822, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/question_duplicate": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0002487639622085348, "nll": 0.007674284037481147, "ece10": 0.007610993376120173, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "jabr_v2/reading_level": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 16, "accuracy": 1.0, "wilson95": [ 0.8063923194655637, 1.0 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00825639436800633, "nll": 0.04593490123388771, "ece10": 0.043555770445894496, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "__failed__": 0 }, "1": { "0": 0, "1": 5, "2": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2" ], "retrospective_majority_baseline": 0.375 }, "jabr_v2/recipe_cuisine": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.018489683811427626, "nll": 0.031703684324349196, "ece10": 0.025241906940157877, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "italian": { "italian": 2, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "french": { "italian": 0, "french": 4, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "mexican": { "italian": 0, "french": 0, "mexican": 3, "indian": 0, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "indian": { "italian": 0, "french": 0, "mexican": 0, "indian": 3, "thai": 0, "middle_eastern": 0, "__failed__": 0 }, "thai": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 2, "middle_eastern": 0, "__failed__": 0 }, "middle_eastern": { "italian": 0, "french": 0, "mexican": 0, "indian": 0, "thai": 0, "middle_eastern": 4, "__failed__": 0 } }, "labels": [ "italian", "french", "mexican", "indian", "thai", "middle_eastern" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/refund_eligible_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.09232565612270431, "nll": 0.1495208672795671, "ece10": 0.08508642100180225, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9411764705882353, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 1, "true": 8, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/return_reason": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0011211149602695326, "nll": 0.014828671811740517, "ece10": 0.014477910919047574, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "damaged_in_transit": { "damaged_in_transit": 3, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "wrong_item": { "damaged_in_transit": 0, "wrong_item": 4, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "size_or_fit": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 3, "changed_mind": 0, "late_delivery": 0, "__failed__": 0 }, "changed_mind": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 4, "late_delivery": 0, "__failed__": 0 }, "late_delivery": { "damaged_in_transit": 0, "wrong_item": 0, "size_or_fit": 0, "changed_mind": 0, "late_delivery": 4, "__failed__": 0 } }, "labels": [ "damaged_in_transit", "wrong_item", "size_or_fit", "changed_mind", "late_delivery" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/review_sentiment_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 13, "accuracy": 0.7222222222222222, "wilson95": [ 0.4912734343970254, 0.8750024662044601 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7222222222222222, "brier": 0.40384830000926125, "nll": 1.012827219030103, "ece10": 0.20120916807984715, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 0.719047619047619, "confusion_matrix": { "0": { "0": 2, "1": 1, "2": 0, "3": 0, "4": 1, "__failed__": 0 }, "1": { "0": 0, "1": 2, "2": 1, "3": 0, "4": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 3, "3": 1, "4": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 3, "4": 0, "__failed__": 0 }, "4": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 3, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3", "4" ], "retrospective_majority_baseline": 0.2222222222222222 }, "jabr_v2/secret_leak_v2": { "expected": 16, "attempted": 16, "complete": true, "n": 16, "correct": 15, "accuracy": 0.9375, "wilson95": [ 0.7167126242970107, 0.9888806552353576 ], "valid": 16, "failures": 0, "coverage": 1.0, "heads": 16, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9375, "brier": 0.10673635674196455, "nll": 0.1787502991667034, "ece10": 0.0703072085644072, "calibration_n": 16, "confident_wrong": 0, "macro_f1": 0.9352226720647774, "confusion_matrix": { "false": { "false": 9, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.625 }, "jabr_v2/sql_injection_risk": { "expected": 15, "attempted": 15, "complete": true, "n": 15, "correct": 15, "accuracy": 1.0, "wilson95": [ 0.7961166989641515, 1.0 ], "valid": 15, "failures": 0, "coverage": 1.0, "heads": 15, "head_correct": 15, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0018090103333897468, "nll": 0.013568535341725219, "ece10": 0.013082621611890829, "calibration_n": 15, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 7, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5333333333333333 }, "jabr_v2/support_department_v2": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.004770094240254131, "nll": 0.02276302596410758, "ece10": 0.02141683776979338, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "billing": { "billing": 5, "tech": 0, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "tech": { "billing": 0, "tech": 3, "sales": 0, "account": 0, "other": 0, "__failed__": 0 }, "sales": { "billing": 0, "tech": 0, "sales": 4, "account": 0, "other": 0, "__failed__": 0 }, "account": { "billing": 0, "tech": 0, "sales": 0, "account": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 0, "tech": 0, "sales": 0, "account": 0, "other": 2, "__failed__": 0 } }, "labels": [ "billing", "tech", "sales", "account", "other" ], "retrospective_majority_baseline": 0.29411764705882354 }, "jabr_v2/suspicious_transaction": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 17, "accuracy": 1.0, "wilson95": [ 0.8156818649911483, 1.0 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0020060006649474734, "nll": 0.025223196137681883, "ece10": 0.024700242582928467, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5294117647058824 }, "jabr_v2/symptom_triage": { "expected": 19, "attempted": 19, "complete": true, "n": 19, "correct": 13, "accuracy": 0.6842105263157895, "wilson95": [ 0.46010407675327475, 0.8463562091220773 ], "valid": 19, "failures": 0, "coverage": 1.0, "heads": 19, "head_correct": 13, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.6842105263157895, "brier": 0.38956301335836674, "nll": 0.5792925419603568, "ece10": 0.21449432662675794, "calibration_n": 19, "confident_wrong": 1, "macro_f1": 0.6130952380952381, "confusion_matrix": { "0": { "0": 7, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 4, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 1, "2": 1, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 1, "3": 4, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.3684210526315789 }, "jabr_v2/travel_policy_violation": { "expected": 17, "attempted": 17, "complete": true, "n": 17, "correct": 16, "accuracy": 0.9411764705882353, "wilson95": [ 0.7301796936242461, 0.9895395990150025 ], "valid": 17, "failures": 0, "coverage": 1.0, "heads": 17, "head_correct": 16, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9411764705882353, "brier": 0.03897247161187858, "nll": 0.07256960076752333, "ece10": 0.05655248061370166, "calibration_n": 17, "confident_wrong": 0, "macro_f1": 0.9377289377289377, "confusion_matrix": { "false": { "false": 10, "true": 1, "__failed__": 0 }, "true": { "false": 0, "true": 6, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.6470588235294118 }, "jabr_v2/urgency_v2": { "expected": 18, "attempted": 18, "complete": true, "n": 18, "correct": 18, "accuracy": 1.0, "wilson95": [ 0.8241207763533425, 1.0 ], "valid": 18, "failures": 0, "coverage": 1.0, "heads": 18, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0006640727139968603, "nll": 0.008425841172552118, "ece10": 0.008253431005993732, "calibration_n": 18, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "false": { "false": 8, "true": 0, "__failed__": 0 }, "true": { "false": 0, "true": 10, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5555555555555556 }, "jabr_v2/veterinary_triage": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 17, "accuracy": 0.85, "wilson95": [ 0.6395811352592431, 0.9476312541037835 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 17, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.85, "brier": 0.1982055497136377, "nll": 0.32215348718133086, "ece10": 0.0926581179439574, "calibration_n": 20, "confident_wrong": 1, "macro_f1": 0.8292540792540792, "confusion_matrix": { "0": { "0": 6, "1": 1, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 4, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 2, "2": 2, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 5, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.35 }, "jabr_v2/voice_assistant_intent": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 20, "accuracy": 1.0, "wilson95": [ 0.8388748419471806, 1.0 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 20, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00016125627995392354, "nll": 0.004157432658195133, "ece10": 0.004111517112531088, "calibration_n": 20, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "device_control": { "device_control": 4, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "information_request": { "device_control": 0, "information_request": 5, "reminder": 0, "music": 0, "small_talk": 0, "__failed__": 0 }, "reminder": { "device_control": 0, "information_request": 0, "reminder": 4, "music": 0, "small_talk": 0, "__failed__": 0 }, "music": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 3, "small_talk": 0, "__failed__": 0 }, "small_talk": { "device_control": 0, "information_request": 0, "reminder": 0, "music": 0, "small_talk": 4, "__failed__": 0 } }, "labels": [ "device_control", "information_request", "reminder", "music", "small_talk" ], "retrospective_majority_baseline": 0.25 }, "jabr_v2/warranty_claim_eligible": { "expected": 20, "attempted": 20, "complete": true, "n": 20, "correct": 18, "accuracy": 0.9, "wilson95": [ 0.6989663547715128, 0.9721335187862319 ], "valid": 20, "failures": 0, "coverage": 1.0, "heads": 20, "head_correct": 18, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9, "brier": 0.11924228268358769, "nll": 0.19054765296735726, "ece10": 0.05564315133820052, "calibration_n": 20, "confident_wrong": 1, "macro_f1": 0.9, "confusion_matrix": { "false": { "false": 9, "true": 0, "__failed__": 0 }, "true": { "false": 2, "true": 9, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.55 }, "jabr_v2/weather_alert_severity": { "expected": 22, "attempted": 22, "complete": true, "n": 22, "correct": 22, "accuracy": 1.0, "wilson95": [ 0.8513451253913791, 1.0 ], "valid": 22, "failures": 0, "coverage": 1.0, "heads": 22, "head_correct": 22, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.023131906709873393, "nll": 0.057209050530988396, "ece10": 0.047977961625903516, "calibration_n": 22, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "0": { "0": 5, "1": 0, "2": 0, "3": 0, "__failed__": 0 }, "1": { "0": 0, "1": 6, "2": 0, "3": 0, "__failed__": 0 }, "2": { "0": 0, "1": 0, "2": 5, "3": 0, "__failed__": 0 }, "3": { "0": 0, "1": 0, "2": 0, "3": 6, "__failed__": 0 } }, "labels": [ "0", "1", "2", "3" ], "retrospective_majority_baseline": 0.2727272727272727 }, "kev_claim/devtools-v1": { "expected": 900, "attempted": 900, "complete": true, "n": 900, "correct": 610, "accuracy": 0.6777777777777778, "wilson95": [ 0.6465462718509619, 0.7074981204227967 ], "valid": 900, "failures": 0, "coverage": 1.0, "heads": 1073, "head_correct": 770, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7176141658900279, "cluster_bootstrap95": [ 0.6382940797208633, 0.7277383903810913 ], "independent_source_groups": 596, "brier": 0.44571700194061475, "nll": 0.9890035855741525, "ece10": 0.18749570836123236, "calibration_n": 1073, "confident_wrong": 141 }, "kev_claim/documents-v1": { "expected": 574, "attempted": 574, "complete": true, "n": 574, "correct": 461, "accuracy": 0.8031358885017421, "wilson95": [ 0.7686374493164543, 0.8336038656460039 ], "valid": 574, "failures": 0, "coverage": 1.0, "heads": 936, "head_correct": 820, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8760683760683761, "brier": 0.19777070298256663, "nll": 0.38934067091817637, "ece10": 0.061682165778640344, "calibration_n": 936, "confident_wrong": 47 }, "kev_claim/hard-v1": { "expected": 700, "attempted": 700, "complete": true, "n": 700, "correct": 428, "accuracy": 0.6114285714285714, "wilson95": [ 0.5748056207960914, 0.6468352019275121 ], "valid": 700, "failures": 0, "coverage": 1.0, "heads": 1088, "head_correct": 769, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7068014705882353, "cluster_bootstrap95": [ 0.5744985673352435, 0.6466423157702493 ], "independent_source_groups": 650, "brier": 0.40688539317895484, "nll": 0.7876917564682085, "ece10": 0.11761275055076839, "calibration_n": 1088, "confident_wrong": 67 }, "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 87, "accuracy": 0.2175, "wilson95": [ 0.1798616207098898, 0.2605128255889877 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 2000, "head_correct": 1398, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.699, "brier": 0.44179708384254474, "nll": 0.7974062927377831, "ece10": 0.1583350721747916, "calibration_n": 2000, "confident_wrong": 177 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.007028019545354514, "nll": 0.01664961738422955, "ece10": 0.01440741842538756, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 27, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9642857142857143, "brier": 0.04754019774605356, "nll": 0.06607808698476857, "ece10": 0.03486801758100478, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 0.9641025641025641, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 1, "purchase": 0, "inspection": 6, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00043039691298404177, "nll": 0.00696073427492266, "ece10": 0.006815308858680528, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "design": { "design": 7, "inspection": 0, "purchase": 0, "work": 0, "__failed__": 0 }, "inspection": { "design": 0, "inspection": 7, "purchase": 0, "work": 0, "__failed__": 0 }, "purchase": { "design": 0, "inspection": 0, "purchase": 7, "work": 0, "__failed__": 0 }, "work": { "design": 0, "inspection": 0, "purchase": 0, "work": 7, "__failed__": 0 } }, "labels": [ "design", "inspection", "purchase", "work" ], "retrospective_majority_baseline": 0.25 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "n": 28, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "valid": 28, "failures": 0, "coverage": 1.0, "heads": 28, "head_correct": 28, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0049350985321275145, "nll": 0.023193888829180703, "ece10": 0.021718474503155154, "calibration_n": 28, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "work": { "work": 7, "purchase": 0, "inspection": 0, "design": 0, "__failed__": 0 }, "purchase": { "work": 0, "purchase": 7, "inspection": 0, "design": 0, "__failed__": 0 }, "inspection": { "work": 0, "purchase": 0, "inspection": 7, "design": 0, "__failed__": 0 }, "design": { "work": 0, "purchase": 0, "inspection": 0, "design": 7, "__failed__": 0 } }, "labels": [ "work", "purchase", "inspection", "design" ], "retrospective_majority_baseline": 0.25 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.0027418711327046345, "nll": 0.01649417111871552, "ece10": 0.015658135222946734, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "review": { "review": 8, "it": 0, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.002734237188665643, "nll": 0.01626181695201149, "ece10": 0.015440867307771423, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 8, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.02143221120756852, "nll": 0.05279117401785215, "ece10": 0.04509813964693623, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "review": { "review": 8, "it": 0, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "it": { "review": 0, "it": 8, "documents": 0, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "documents": { "review": 0, "it": 0, "documents": 8, "procurement": 0, "maintenance": 0, "__failed__": 0 }, "procurement": { "review": 0, "it": 0, "documents": 0, "procurement": 8, "maintenance": 0, "__failed__": 0 }, "maintenance": { "review": 0, "it": 0, "documents": 0, "procurement": 0, "maintenance": 8, "__failed__": 0 } }, "labels": [ "review", "it", "documents", "procurement", "maintenance" ], "retrospective_majority_baseline": 0.2 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "n": 40, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "valid": 40, "failures": 0, "coverage": 1.0, "heads": 40, "head_correct": 40, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 1.0, "brier": 0.00320822154971512, "nll": 0.02265126478621842, "ece10": 0.021635314524283702, "calibration_n": 40, "confident_wrong": 0, "macro_f1": 1.0, "confusion_matrix": { "maintenance": { "maintenance": 8, "procurement": 0, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "procurement": { "maintenance": 0, "procurement": 8, "documents": 0, "it": 0, "review": 0, "__failed__": 0 }, "documents": { "maintenance": 0, "procurement": 0, "documents": 8, "it": 0, "review": 0, "__failed__": 0 }, "it": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 8, "review": 0, "__failed__": 0 }, "review": { "maintenance": 0, "procurement": 0, "documents": 0, "it": 0, "review": 8, "__failed__": 0 } }, "labels": [ "maintenance", "procurement", "documents", "it", "review" ], "retrospective_majority_baseline": 0.2 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 246, "accuracy": 0.615, "wilson95": [ 0.5664353568701253, 0.6613768154329864 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 246, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.615, "brier": 0.646616239138382, "nll": 1.6607680404319212, "ece10": 0.28799831071279414, "calibration_n": 400, "confident_wrong": 85, "macro_f1": 0.5263639513328562, "confusion_matrix": { "sadness": { "sadness": 93, "joy": 9, "love": 5, "anger": 3, "fear": 9, "surprise": 0, "__failed__": 0 }, "joy": { "sadness": 25, "joy": 70, "love": 16, "anger": 5, "fear": 1, "surprise": 1, "__failed__": 0 }, "love": { "sadness": 10, "joy": 9, "love": 13, "anger": 3, "fear": 2, "surprise": 0, "__failed__": 0 }, "anger": { "sadness": 19, "joy": 4, "love": 4, "anger": 33, "fear": 1, "surprise": 3, "__failed__": 0 }, "fear": { "sadness": 11, "joy": 0, "love": 0, "anger": 4, "fear": 35, "surprise": 2, "__failed__": 0 }, "surprise": { "sadness": 2, "joy": 1, "love": 1, "anger": 1, "fear": 3, "surprise": 2, "__failed__": 0 } }, "labels": [ "sadness", "joy", "love", "anger", "fear", "surprise" ], "retrospective_majority_baseline": 0.2975 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 271, "accuracy": 0.9033333333333333, "wilson95": [ 0.8646182980304887, 0.9318497051817106 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 271, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.9033333333333333, "brier": 0.1468437728859605, "nll": 0.3809367695253286, "ece10": 0.056915852261092574, "calibration_n": 300, "confident_wrong": 9 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 378, "accuracy": 0.945, "wilson95": [ 0.9181325521920267, 0.9634015058504493 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 378, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.945, "brier": 0.09203521199470649, "nll": 0.16383235195865456, "ece10": 0.02695094516007608, "calibration_n": 400, "confident_wrong": 9, "macro_f1": 0.9211469534050178, "confusion_matrix": { "false": { "false": 299, "true": 10, "__failed__": 0 }, "true": { "false": 12, "true": 79, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.7725 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 337, "accuracy": 0.8425, "wilson95": [ 0.8035652075632727, 0.8749188708177342 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 337, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8425, "brier": 0.2679042449142313, "nll": 0.7309923562936456, "ece10": 0.1259325738459009, "calibration_n": 400, "confident_wrong": 40, "macro_f1": 0.8294153714453135, "confusion_matrix": { "world": { "world": 86, "sports": 6, "business": 10, "sci_tech": 1, "__failed__": 0 }, "sports": { "world": 3, "sports": 120, "business": 0, "sci_tech": 0, "__failed__": 0 }, "business": { "world": 3, "sports": 0, "business": 66, "sci_tech": 3, "__failed__": 0 }, "sci_tech": { "world": 11, "sports": 0, "business": 26, "sci_tech": 65, "__failed__": 0 } }, "labels": [ "world", "sports", "business", "sci_tech" ], "retrospective_majority_baseline": 0.3075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 353, "accuracy": 0.8825, "wilson95": [ 0.8472450357843669, 0.9104780590498961 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 353, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8825, "brier": 0.18947221873753617, "nll": 0.3292291725076886, "ece10": 0.0797460095570678, "calibration_n": 400, "confident_wrong": 27, "macro_f1": 0.8702293764797648, "confusion_matrix": { "true": { "true": 115, "false": 38, "__failed__": 0 }, "false": { "true": 9, "false": 238, "__failed__": 0 } }, "labels": [ "true", "false" ], "retrospective_majority_baseline": 0.6175 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "n": 300, "correct": 224, "accuracy": 0.7466666666666667, "wilson95": [ 0.6945458844004171, 0.7925502497954735 ], "valid": 300, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 224, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7466666666666667, "brier": 0.4536619542080065, "nll": 1.3472797056811754, "ece10": 0.20320596262309001, "calibration_n": 300, "confident_wrong": 53 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 388, "accuracy": 0.97, "wilson95": [ 0.9483014642909042, 0.9827569790348569 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 388, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.97, "brier": 0.05377450014281278, "nll": 0.0987094070807882, "ece10": 0.010508907637229888, "calibration_n": 400, "confident_wrong": 6, "macro_f1": 0.9699519230769231, "confusion_matrix": { "false": { "false": 186, "true": 2, "__failed__": 0 }, "true": { "false": 10, "true": 202, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.53 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "n": 400, "correct": 283, "accuracy": 0.7075, "wilson95": [ 0.6611144240081159, 0.7499379738431511 ], "valid": 400, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 283, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7075, "brier": 0.4492189357604921, "nll": 0.7527900193491519, "ece10": 0.2018162742377221, "calibration_n": 400, "confident_wrong": 60, "macro_f1": 0.6867281161515755, "confusion_matrix": { "false": { "false": 193, "true": 7, "__failed__": 0 }, "true": { "false": 110, "true": 90, "__failed__": 0 } }, "labels": [ "false", "true" ], "retrospective_majority_baseline": 0.5 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 78, "accuracy": 0.78, "wilson95": [ 0.6892964648501421, 0.8499871761539459 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 78, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.78, "brier": 0.42463080290827343, "nll": 0.9876227502809393, "ece10": 0.206280843353827, "calibration_n": 100, "confident_wrong": 21, "macro_f1": 0.7612847222222222, "confusion_matrix": { "yes": { "yes": 53, "no": 1, "__failed__": 0 }, "no": { "yes": 21, "no": 25, "__failed__": 0 } }, "labels": [ "yes", "no" ], "retrospective_majority_baseline": 0.54 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 47, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47, "brier": 0.8779641639818007, "nll": 2.726163138493714, "ece10": 0.41142834950601176, "calibration_n": 100, "confident_wrong": 27, "macro_f1": 0.510782967032967, "confusion_matrix": { "balance": { "balance": 3, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "transfer": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "card_lost": { "balance": 0, "transfer": 0, "card_lost": 2, "dispute_charge": 2, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "dispute_charge": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "loan_payment": { "balance": 1, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 2, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "mortgage": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 6, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "wire": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 1, "loan_payment": 1, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "statement": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 2, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 4, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fraud_alert": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 1, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 5, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "branch_appointment": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 4, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "fee_refund": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 7, "loan_payment": 0, "mortgage": 0, "wire": 1, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 5, "direct_deposit": 0, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "direct_deposit": { "balance": 0, "transfer": 0, "card_lost": 1, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 3, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 1, "__failed__": 0 }, "close_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 0, "close_account": 5, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 }, "new_account": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 1, "loan_payment": 0, "mortgage": 0, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 2, "limit_increase": 0, "other": 1, "__failed__": 0 }, "limit_increase": { "balance": 0, "transfer": 0, "card_lost": 0, "dispute_charge": 0, "loan_payment": 0, "mortgage": 1, "wire": 0, "statement": 0, "fraud_alert": 0, "branch_appointment": 0, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 4, "other": 0, "__failed__": 0 }, "other": { "balance": 0, "transfer": 1, "card_lost": 0, "dispute_charge": 4, "loan_payment": 0, "mortgage": 0, "wire": 2, "statement": 0, "fraud_alert": 0, "branch_appointment": 1, "fee_refund": 0, "direct_deposit": 1, "close_account": 0, "new_account": 0, "limit_increase": 0, "other": 0, "__failed__": 0 } }, "labels": [ "balance", "transfer", "card_lost", "dispute_charge", "loan_payment", "mortgage", "wire", "statement", "fraud_alert", "branch_appointment", "fee_refund", "direct_deposit", "close_account", "new_account", "limit_increase", "other" ], "retrospective_majority_baseline": 0.13 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 53, "accuracy": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 152, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.4230220817032375, "nll": 0.808585822046449, "ece10": 0.19960594591154182, "calibration_n": 200, "confident_wrong": 35 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 133, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.665, "brier": 0.5961260499275436, "nll": 1.4811345290847842, "ece10": 0.2587472200724777, "calibration_n": 200, "confident_wrong": 47 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 76, "accuracy": 0.76, "wilson95": [ 0.6676766365018553, 0.8330867444305122 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 76, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.76, "brier": 0.395660715878919, "nll": 1.1320995237229114, "ece10": 0.19042033690106708, "calibration_n": 100, "confident_wrong": 13, "macro_f1": 0.7044708813698352, "confusion_matrix": { "invoice": { "invoice": 7, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "receipt": { "invoice": 1, "receipt": 13, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "contract": { "invoice": 0, "receipt": 0, "contract": 6, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "resume": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 4, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 1, "__failed__": 0 }, "bank_statement": { "invoice": 2, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 12, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "clinical_note": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 6, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 3, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "syllabus": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 5, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "news_article": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 1, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 3, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "lease": { "invoice": 1, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 8, "research_abstract": 0, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "research_abstract": { "invoice": 0, "receipt": 0, "contract": 1, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 6, "recipe": 0, "sports_recap": 0, "__failed__": 0 }, "recipe": { "invoice": 0, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 1, "syllabus": 1, "news_article": 0, "lease": 0, "research_abstract": 1, "recipe": 3, "sports_recap": 0, "__failed__": 0 }, "sports_recap": { "invoice": 2, "receipt": 0, "contract": 0, "resume": 0, "bank_statement": 0, "clinical_note": 0, "syllabus": 0, "news_article": 0, "lease": 0, "research_abstract": 0, "recipe": 0, "sports_recap": 6, "__failed__": 0 } }, "labels": [ "invoice", "receipt", "contract", "resume", "bank_statement", "clinical_note", "syllabus", "news_article", "lease", "research_abstract", "recipe", "sports_recap" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 400, "head_correct": 219, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5475, "brier": 0.804941599036983, "nll": 2.232354428315263, "ece10": 0.37361317208027545, "calibration_n": 400, "confident_wrong": 127 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 67, "accuracy": 0.67, "wilson95": [ 0.5730534379509052, 0.7543687726587197 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 67, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.67, "brier": 0.5496644329179731, "nll": 1.7488731512360038, "ece10": 0.2232262282448531, "calibration_n": 100, "confident_wrong": 14, "macro_f1": 0.6716556972439325, "confusion_matrix": { "politics": { "politics": 4, "business": 0, "technology": 1, "science": 1, "health": 3, "sports": 0, "entertainment": 1, "world": 3, "culture": 0, "weather": 1, "__failed__": 0 }, "business": { "politics": 0, "business": 9, "technology": 0, "science": 0, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 0, "__failed__": 0 }, "technology": { "politics": 0, "business": 1, "technology": 8, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "science": { "politics": 1, "business": 1, "technology": 0, "science": 6, "health": 0, "sports": 0, "entertainment": 0, "world": 1, "culture": 0, "weather": 1, "__failed__": 0 }, "health": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 10, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "sports": { "politics": 1, "business": 0, "technology": 0, "science": 2, "health": 0, "sports": 5, "entertainment": 0, "world": 0, "culture": 0, "weather": 0, "__failed__": 0 }, "entertainment": { "politics": 1, "business": 1, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 5, "world": 0, "culture": 2, "weather": 0, "__failed__": 0 }, "world": { "politics": 1, "business": 0, "technology": 1, "science": 0, "health": 0, "sports": 0, "entertainment": 1, "world": 6, "culture": 0, "weather": 0, "__failed__": 0 }, "culture": { "politics": 0, "business": 0, "technology": 0, "science": 0, "health": 1, "sports": 0, "entertainment": 0, "world": 0, "culture": 8, "weather": 1, "__failed__": 0 }, "weather": { "politics": 0, "business": 2, "technology": 0, "science": 1, "health": 0, "sports": 0, "entertainment": 0, "world": 0, "culture": 0, "weather": 6, "__failed__": 0 } }, "labels": [ "politics", "business", "technology", "science", "health", "sports", "entertainment", "world", "culture", "weather" ], "retrospective_majority_baseline": 0.14 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 47, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.47, "brier": 0.9448716938846556, "nll": 2.8824976306377836, "ece10": 0.38069747475013777, "calibration_n": 100, "confident_wrong": 26, "macro_f1": 0.443498361287835, "confusion_matrix": { "biology": { "biology": 8, "medicine": 0, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "medicine": { "biology": 1, "medicine": 9, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "chemistry": { "biology": 1, "medicine": 0, "chemistry": 5, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "physics": { "biology": 4, "medicine": 2, "chemistry": 0, "physics": 6, "computer_science": 0, "mathematics": 0, "earth_science": 2, "engineering": 2, "economics": 0, "other": 0, "__failed__": 0 }, "computer_science": { "biology": 2, "medicine": 1, "chemistry": 1, "physics": 0, "computer_science": 3, "mathematics": 0, "earth_science": 2, "engineering": 4, "economics": 1, "other": 1, "__failed__": 0 }, "mathematics": { "biology": 0, "medicine": 1, "chemistry": 2, "physics": 1, "computer_science": 0, "mathematics": 1, "earth_science": 1, "engineering": 0, "economics": 0, "other": 0, "__failed__": 0 }, "earth_science": { "biology": 2, "medicine": 3, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 5, "engineering": 1, "economics": 0, "other": 0, "__failed__": 0 }, "engineering": { "biology": 0, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 0, "engineering": 6, "economics": 0, "other": 0, "__failed__": 0 }, "economics": { "biology": 2, "medicine": 1, "chemistry": 0, "physics": 0, "computer_science": 0, "mathematics": 0, "earth_science": 1, "engineering": 0, "economics": 4, "other": 0, "__failed__": 0 }, "other": { "biology": 3, "medicine": 0, "chemistry": 0, "physics": 2, "computer_science": 1, "mathematics": 1, "earth_science": 2, "engineering": 3, "economics": 0, "other": 0, "__failed__": 0 } }, "labels": [ "biology", "medicine", "chemistry", "physics", "computer_science", "mathematics", "earth_science", "engineering", "economics", "other" ], "retrospective_majority_baseline": 0.16 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 14, "accuracy": 0.14, "wilson95": [ 0.08526343659939134, 0.22137188210963837 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 74, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.37, "brier": 0.8043463613852248, "nll": 2.2827351731108267, "ece10": 0.4031997258620558, "calibration_n": 100, "confident_wrong": 37 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 37, "accuracy": 0.37, "wilson95": [ 0.28182360534324524, 0.4677947041905709 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 129, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.645, "brier": 0.300728499887479, "nll": 0.6258284814863935, "ece10": 0.1213858827836401, "calibration_n": 100, "confident_wrong": 10 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 84, "accuracy": 0.84, "wilson95": [ 0.7557973061072976, 0.8990471151119521 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 84, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.84, "brier": 0.26291211350857674, "nll": 0.5700030102687318, "ece10": 0.11350070465791372, "calibration_n": 100, "confident_wrong": 9, "macro_f1": 0.8423231473694047, "confusion_matrix": { "negative": { "negative": 34, "neutral": 2, "positive": 0, "__failed__": 0 }, "neutral": { "negative": 10, "neutral": 23, "positive": 0, "__failed__": 0 }, "positive": { "negative": 2, "neutral": 2, "positive": 27, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.36 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 19, "accuracy": 0.19, "wilson95": [ 0.12514751509768735, 0.27778845379064376 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 200, "head_correct": 103, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.515, "brier": 0.2723017533850625, "nll": 0.4176498588612247, "ece10": 0.11364555924207775, "calibration_n": 100, "confident_wrong": 5 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 243, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.81, "brier": 0.3384186689709526, "nll": 0.8151760532667373, "ece10": 0.14860944760624528, "calibration_n": 300, "confident_wrong": 43 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 61, "accuracy": 0.61, "wilson95": [ 0.5120301704584073, 0.6998312599360558 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 61, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.61, "brier": 0.6225469169210248, "nll": 2.156498019609119, "ece10": 0.2243050575337471, "calibration_n": 100, "confident_wrong": 15, "macro_f1": 0.5425372246800818, "confusion_matrix": { "order_status": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "refund_request": { "order_status": 0, "refund_request": 4, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "cancel_subscription": { "order_status": 0, "refund_request": 0, "cancel_subscription": 2, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "update_payment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 3, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "login_problem": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 3, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "password_reset": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 1, "password_reset": 4, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "shipping_delay": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "missing_item": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "damaged_item": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 2, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_address": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 5, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "change_order": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 1, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 1, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "product_question": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 1, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "pricing_question": { "order_status": 0, "refund_request": 2, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "upgrade_plan": { "order_status": 1, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "downgrade_plan": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 4, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "bug_report": { "order_status": 0, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "feature_request": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 2, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 1, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 1, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "speak_to_human": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 2, "feature_request": 0, "speak_to_human": 3, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "complaint": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 1, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 1, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "invoice_copy": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 8, "tax_document": 1, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "tax_document": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 1, "tax_document": 4, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "appointment": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 1, "upgrade_plan": 0, "downgrade_plan": 1, "bug_report": 0, "feature_request": 0, "speak_to_human": 1, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 4, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "outage": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "data_export": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 1, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 1, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "close_account": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 1, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "promo_code": { "order_status": 2, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 }, "partner_inquiry": { "order_status": 1, "refund_request": 1, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 0, "upgrade_plan": 1, "downgrade_plan": 0, "bug_report": 0, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 2, "other": 0, "__failed__": 0 }, "other": { "order_status": 0, "refund_request": 0, "cancel_subscription": 0, "update_payment": 0, "login_problem": 0, "password_reset": 0, "shipping_delay": 0, "missing_item": 0, "damaged_item": 0, "change_address": 0, "change_order": 0, "product_question": 0, "pricing_question": 2, "upgrade_plan": 0, "downgrade_plan": 0, "bug_report": 1, "feature_request": 0, "speak_to_human": 0, "complaint": 0, "invoice_copy": 0, "tax_document": 0, "appointment": 0, "outage": 0, "data_export": 0, "close_account": 0, "promo_code": 0, "partner_inquiry": 0, "other": 0, "__failed__": 0 } }, "labels": [ "order_status", "refund_request", "cancel_subscription", "update_payment", "login_problem", "password_reset", "shipping_delay", "missing_item", "damaged_item", "change_address", "change_order", "product_question", "pricing_question", "upgrade_plan", "downgrade_plan", "bug_report", "feature_request", "speak_to_human", "complaint", "invoice_copy", "tax_document", "appointment", "outage", "data_export", "close_account", "promo_code", "partner_inquiry", "other" ], "retrospective_majority_baseline": 0.09 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 42, "accuracy": 0.42, "wilson95": [ 0.3279838267435473, 0.5179351329695703 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 42, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.42, "brier": 1.0393190671484385, "nll": 3.043262388396029, "ece10": 0.5003387816558424, "calibration_n": 100, "confident_wrong": 36, "macro_f1": 0.381244064577398, "confusion_matrix": { "billing": { "billing": 7, "account_access": 1, "shipping": 0, "returns": 2, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "account_access": { "billing": 2, "account_access": 8, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "shipping": { "billing": 0, "account_access": 0, "shipping": 3, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 3, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "returns": { "billing": 2, "account_access": 0, "shipping": 0, "returns": 5, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "technical_outage": { "billing": 4, "account_access": 2, "shipping": 0, "returns": 0, "technical_outage": 2, "onboarding": 0, "integrations": 3, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "onboarding": { "billing": 1, "account_access": 1, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 1, "integrations": 3, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "integrations": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 7, "security": 0, "compliance": 0, "pricing": 0, "data_privacy": 0, "other": 0, "__failed__": 0 }, "security": { "billing": 0, "account_access": 1, "shipping": 0, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 2, "security": 2, "compliance": 0, "pricing": 0, "data_privacy": 2, "other": 0, "__failed__": 0 }, "compliance": { "billing": 1, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 1, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 2, "pricing": 1, "data_privacy": 1, "other": 0, "__failed__": 0 }, "pricing": { "billing": 3, "account_access": 0, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 0, "compliance": 0, "pricing": 2, "data_privacy": 0, "other": 0, "__failed__": 0 }, "data_privacy": { "billing": 0, "account_access": 2, "shipping": 0, "returns": 1, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 0, "data_privacy": 3, "other": 0, "__failed__": 0 }, "other": { "billing": 4, "account_access": 0, "shipping": 1, "returns": 0, "technical_outage": 0, "onboarding": 0, "integrations": 0, "security": 1, "compliance": 0, "pricing": 1, "data_privacy": 2, "other": 0, "__failed__": 0 } }, "labels": [ "billing", "account_access", "shipping", "returns", "technical_outage", "onboarding", "integrations", "security", "compliance", "pricing", "data_privacy", "other" ], "retrospective_majority_baseline": 0.11 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 13, "accuracy": 0.13, "wilson95": [ 0.07757167427240512, 0.20980351440076428 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 300, "head_correct": 153, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.51, "brier": 0.8133786679571228, "nll": 2.364792359781912, "ece10": 0.352816207201859, "calibration_n": 300, "confident_wrong": 58 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 56, "accuracy": 0.56, "wilson95": [ 0.4622810465167699, 0.6532797336983921 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 56, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.56, "brier": 0.7452270138375288, "nll": 2.4409111884772714, "ece10": 0.3578820836662544, "calibration_n": 100, "confident_wrong": 25, "macro_f1": 0.5324460627092206, "confusion_matrix": { "new_booking": { "new_booking": 1, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "change_flight": { "new_booking": 0, "change_flight": 5, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 1, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 1, "__failed__": 0 }, "cancel_trip": { "new_booking": 0, "change_flight": 0, "cancel_trip": 2, "seat": 0, "baggage": 0, "refund": 1, "hotel": 1, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "seat": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 7, "baggage": 0, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "baggage": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 1, "baggage": 11, "refund": 0, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "refund": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 6, "hotel": 0, "car": 1, "visa_docs": 0, "loyalty": 0, "disruption": 0, "other": 0, "__failed__": 0 }, "hotel": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 9, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 1, "__failed__": 0 }, "car": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 0, "car": 3, "visa_docs": 0, "loyalty": 0, "disruption": 1, "other": 0, "__failed__": 0 }, "visa_docs": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 3, "car": 0, "visa_docs": 2, "loyalty": 1, "disruption": 0, "other": 1, "__failed__": 0 }, "loyalty": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 1, "loyalty": 5, "disruption": 1, "other": 0, "__failed__": 0 }, "disruption": { "new_booking": 0, "change_flight": 0, "cancel_trip": 0, "seat": 0, "baggage": 0, "refund": 0, "hotel": 2, "car": 0, "visa_docs": 0, "loyalty": 0, "disruption": 5, "other": 0, "__failed__": 0 }, "other": { "new_booking": 0, "change_flight": 1, "cancel_trip": 0, "seat": 0, "baggage": 1, "refund": 0, "hotel": 3, "car": 2, "visa_docs": 1, "loyalty": 1, "disruption": 3, "other": 0, "__failed__": 0 } }, "labels": [ "new_booking", "change_flight", "cancel_trip", "seat", "baggage", "refund", "hotel", "car", "visa_docs", "loyalty", "disruption", "other" ], "retrospective_majority_baseline": 0.13 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 89, "accuracy": 0.89, "wilson95": [ 0.8136870349691969, 0.9374580364293543 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 89, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.89, "brier": 0.19193958912639433, "nll": 0.40857870024761384, "ece10": 0.08500571451094184, "calibration_n": 100, "confident_wrong": 7, "macro_f1": 0.8827328101051741, "confusion_matrix": { "business": { "business": 24, "science and technology": 0, "sports": 0, "world": 1, "__failed__": 0 }, "science and technology": { "business": 7, "science and technology": 15, "sports": 1, "world": 2, "__failed__": 0 }, "sports": { "business": 0, "science and technology": 0, "sports": 25, "world": 0, "__failed__": 0 }, "world": { "business": 0, "science and technology": 0, "sports": 0, "world": 25, "__failed__": 0 } }, "labels": [ "business", "science and technology", "sports", "world" ], "retrospective_majority_baseline": 0.25 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "valid": 0, "failures": 100, "coverage": 0.0, "heads": 100, "head_correct": 0, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.0, "macro_f1": 0.0, "confusion_matrix": { "refund not showing up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "activating a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "age limits": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "apple pay or google pay": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "ATM support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "automatic top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "balance not updated after a cheque or cash deposit": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "beneficiary not being allowed or a failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cancellation of a transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "expiration of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "scope of acceptance of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "arrival of a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card delivery estimate or timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card payment fee": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "payment the customer does not recognise": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "wrong exchange rate": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "card swallowed by a machine": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "cash withdrawal charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "changing a pin code": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "compromised card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "contactless not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "country-specific support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "declined transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 2 }, "unrecognised direct debit payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "limits of disposable cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "editing personal details": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rate charges": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "exchange rates": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "extra charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "failed transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "fiat currency support": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a disposable virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a spare card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "getting a virtual card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "lost or stolen phone": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "ordering a card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "forgotten passcode": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending top up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "pending transfer": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "blocked pin": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "receiving money": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "refund request": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "reverting a card payment": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "terminating an account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "charge for topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by cash or cheque": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up issues or failures": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "top up limitations": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issues with topping up": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "topping up by card": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transaction charged twice": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with a transfer fee charge": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfers into the customer's own account": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer that has not arrived yet": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "transfer timing": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "issue with identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "identity verification": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "source of funds": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "verification and top ups": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "virtual card not working": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "types of bank cards": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "why identity verification is necessary": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong amount of cash received": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 }, "wrong exchange rate for a cash withdrawal": { "refund not showing up": 0, "activating a card": 0, "age limits": 0, "apple pay or google pay": 0, "ATM support": 0, "automatic top up": 0, "balance not updated after a transfer": 0, "balance not updated after a cheque or cash deposit": 0, "beneficiary not being allowed or a failed transfer": 0, "cancellation of a transfer": 0, "expiration of a card": 0, "scope of acceptance of a card": 0, "arrival of a card": 0, "card delivery estimate or timing": 0, "card not working": 0, "card payment fee": 0, "payment the customer does not recognise": 0, "wrong exchange rate": 0, "card swallowed by a machine": 0, "cash withdrawal charge": 0, "unrecognised cash withdrawal": 0, "changing a pin code": 0, "compromised card": 0, "contactless not working": 0, "country-specific support": 0, "declined card payment": 0, "declined cash withdrawal": 0, "declined transfer": 0, "unrecognised direct debit payment": 0, "limits of disposable cards": 0, "editing personal details": 0, "exchange rate charges": 0, "exchange rates": 0, "extra charge": 0, "failed transfer": 0, "fiat currency support": 0, "getting a disposable virtual card": 0, "getting a spare card": 0, "getting a virtual card": 0, "lost or stolen card": 0, "lost or stolen phone": 0, "ordering a card": 0, "forgotten passcode": 0, "pending card payment": 0, "pending cash withdrawal": 0, "pending top up": 0, "pending transfer": 0, "blocked pin": 0, "receiving money": 0, "refund request": 0, "reverting a card payment": 0, "terminating an account": 0, "charge for topping up by card": 0, "topping up by cash or cheque": 0, "top up issues or failures": 0, "top up limitations": 0, "issues with topping up": 0, "topping up by card": 0, "transaction charged twice": 0, "issue with a transfer fee charge": 0, "transfers into the customer's own account": 0, "transfer that has not arrived yet": 0, "transfer timing": 0, "issue with identity verification": 0, "identity verification": 0, "source of funds": 0, "verification and top ups": 0, "virtual card not working": 0, "types of bank cards": 0, "why identity verification is necessary": 0, "wrong amount of cash received": 0, "wrong exchange rate for a cash withdrawal": 0, "__failed__": 1 } }, "labels": [ "refund not showing up", "activating a card", "age limits", "apple pay or google pay", "ATM support", "automatic top up", "balance not updated after a transfer", "balance not updated after a cheque or cash deposit", "beneficiary not being allowed or a failed transfer", "cancellation of a transfer", "expiration of a card", "scope of acceptance of a card", "arrival of a card", "card delivery estimate or timing", "card not working", "card payment fee", "payment the customer does not recognise", "wrong exchange rate", "card swallowed by a machine", "cash withdrawal charge", "unrecognised cash withdrawal", "changing a pin code", "compromised card", "contactless not working", "country-specific support", "declined card payment", "declined cash withdrawal", "declined transfer", "unrecognised direct debit payment", "limits of disposable cards", "editing personal details", "exchange rate charges", "exchange rates", "extra charge", "failed transfer", "fiat currency support", "getting a disposable virtual card", "getting a spare card", "getting a virtual card", "lost or stolen card", "lost or stolen phone", "ordering a card", "forgotten passcode", "pending card payment", "pending cash withdrawal", "pending top up", "pending transfer", "blocked pin", "receiving money", "refund request", "reverting a card payment", "terminating an account", "charge for topping up by card", "topping up by cash or cheque", "top up issues or failures", "top up limitations", "issues with topping up", "topping up by card", "transaction charged twice", "issue with a transfer fee charge", "transfers into the customer's own account", "transfer that has not arrived yet", "transfer timing", "issue with identity verification", "identity verification", "source of funds", "verification and top ups", "virtual card not working", "types of bank cards", "why identity verification is necessary", "wrong amount of cash received", "wrong exchange rate for a cash withdrawal" ], "retrospective_majority_baseline": 0.02 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "n": 100, "correct": 50, "accuracy": 0.5, "wilson95": [ 0.4038315303659956, 0.5961684696340044 ], "valid": 100, "failures": 0, "coverage": 1.0, "heads": 100, "head_correct": 50, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.5, "brier": 0.8541805000575795, "nll": 2.273381289826863, "ece10": 0.38250409008205877, "calibration_n": 100, "confident_wrong": 23, "macro_f1": 0.49424591385375694, "confusion_matrix": { "anger": { "anger": 9, "fear": 4, "joy": 0, "love": 0, "sadness": 4, "surprise": 0, "__failed__": 0 }, "fear": { "anger": 1, "fear": 11, "joy": 0, "love": 0, "sadness": 4, "surprise": 1, "__failed__": 0 }, "joy": { "anger": 1, "fear": 0, "joy": 9, "love": 1, "sadness": 4, "surprise": 2, "__failed__": 0 }, "love": { "anger": 1, "fear": 1, "joy": 2, "love": 7, "sadness": 6, "surprise": 0, "__failed__": 0 }, "sadness": { "anger": 0, "fear": 2, "joy": 2, "love": 1, "sadness": 11, "surprise": 0, "__failed__": 0 }, "surprise": { "anger": 1, "fear": 0, "joy": 5, "love": 1, "sadness": 6, "surprise": 3, "__failed__": 0 } }, "labels": [ "anger", "fear", "joy", "love", "sadness", "surprise" ], "retrospective_majority_baseline": 0.17 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2454, "accuracy": 0.8324287652645862, "wilson95": [ 0.8185158794212413, 0.8454764207556964 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2454, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.8324287652645862, "brier": 0.27256065355505155, "nll": 0.7255697436512477, "ece10": 0.10500015236425694, "calibration_n": 2948, "confident_wrong": 222, "macro_f1": 0.83777136744738, "confusion_matrix": { "alarm": { "alarm": 91, "audio": 0, "calendar": 3, "cooking": 0, "datetime": 1, "email": 0, "general": 0, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 47, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 10, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 2, "audio": 0, "calendar": 351, "cooking": 0, "datetime": 9, "email": 0, "general": 14, "iot": 0, "lists": 5, "music": 0, "news": 2, "play": 0, "qa": 10, "recommendation": 5, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 67, "datetime": 0, "email": 0, "general": 2, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 1, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 84, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 15, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "email": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 222, "general": 27, "iot": 0, "lists": 3, "music": 0, "news": 0, "play": 0, "qa": 14, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 3, "datetime": 0, "email": 0, "general": 114, "iot": 3, "lists": 0, "music": 1, "news": 0, "play": 2, "qa": 47, "recommendation": 8, "social": 0, "takeaway": 1, "transport": 2, "weather": 2, "__failed__": 0 }, "iot": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 13, "iot": 200, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 0, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 9, "iot": 2, "lists": 124, "music": 2, "news": 0, "play": 0, "qa": 2, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 1, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 0, "lists": 1, "music": 64, "news": 0, "play": 3, "qa": 4, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "news": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 8, "iot": 0, "lists": 0, "music": 0, "news": 93, "play": 1, "qa": 20, "recommendation": 0, "social": 1, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "play": { "alarm": 0, "audio": 3, "calendar": 0, "cooking": 0, "datetime": 2, "email": 0, "general": 14, "iot": 5, "lists": 0, "music": 78, "news": 1, "play": 276, "qa": 6, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 9, "iot": 0, "lists": 0, "music": 0, "news": 1, "play": 0, "qa": 271, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 2, "weather": 3, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 4, "play": 1, "qa": 7, "recommendation": 76, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 22, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 5, "recommendation": 0, "social": 77, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 0, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 6, "recommendation": 1, "social": 0, "takeaway": 41, "transport": 1, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 5, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 117, "weather": 0, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 7, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 139, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "n": 2948, "correct": 2299, "accuracy": 0.7798507462686567, "wilson95": [ 0.7645347093754712, 0.7944384004673589 ], "valid": 2948, "failures": 0, "coverage": 1.0, "heads": 2948, "head_correct": 2299, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7798507462686567, "brier": 0.3464341225417169, "nll": 0.9550316214488618, "ece10": 0.1355050026539863, "calibration_n": 2948, "confident_wrong": 225, "macro_f1": 0.7951805816927305, "confusion_matrix": { "alarm": { "alarm": 90, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 1, "email": 0, "general": 1, "iot": 1, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "audio": { "alarm": 1, "audio": 43, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 12, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 0, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "calendar": { "alarm": 3, "audio": 0, "calendar": 339, "cooking": 0, "datetime": 3, "email": 1, "general": 22, "iot": 1, "lists": 6, "music": 0, "news": 1, "play": 0, "qa": 14, "recommendation": 7, "social": 1, "takeaway": 1, "transport": 0, "weather": 1, "__failed__": 0 }, "cooking": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 64, "datetime": 0, "email": 0, "general": 3, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 2, "recommendation": 0, "social": 0, "takeaway": 2, "transport": 0, "weather": 0, "__failed__": 0 }, "datetime": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 83, "email": 0, "general": 0, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 17, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "email": { "alarm": 1, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 219, "general": 25, "iot": 3, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 15, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 1, "weather": 2, "__failed__": 0 }, "general": { "alarm": 0, "audio": 0, "calendar": 4, "cooking": 3, "datetime": 1, "email": 1, "general": 117, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 1, "qa": 45, "recommendation": 5, "social": 0, "takeaway": 1, "transport": 3, "weather": 3, "__failed__": 0 }, "iot": { "alarm": 1, "audio": 4, "calendar": 0, "cooking": 1, "datetime": 0, "email": 0, "general": 22, "iot": 183, "lists": 1, "music": 0, "news": 0, "play": 0, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "lists": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 12, "iot": 2, "lists": 120, "music": 3, "news": 0, "play": 1, "qa": 1, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 0, "weather": 0, "__failed__": 0 }, "music": { "alarm": 0, "audio": 0, "calendar": 1, "cooking": 0, "datetime": 0, "email": 0, "general": 9, "iot": 0, "lists": 1, "music": 59, "news": 0, "play": 2, "qa": 7, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "news": { "alarm": 1, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 15, "iot": 0, "lists": 1, "music": 0, "news": 80, "play": 0, "qa": 24, "recommendation": 1, "social": 1, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "play": { "alarm": 1, "audio": 2, "calendar": 2, "cooking": 0, "datetime": 3, "email": 1, "general": 34, "iot": 14, "lists": 4, "music": 69, "news": 0, "play": 225, "qa": 27, "recommendation": 2, "social": 0, "takeaway": 0, "transport": 0, "weather": 1, "__failed__": 0 }, "qa": { "alarm": 0, "audio": 1, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 20, "iot": 1, "lists": 0, "music": 1, "news": 1, "play": 0, "qa": 257, "recommendation": 1, "social": 0, "takeaway": 0, "transport": 2, "weather": 2, "__failed__": 0 }, "recommendation": { "alarm": 0, "audio": 0, "calendar": 2, "cooking": 0, "datetime": 0, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 0, "news": 2, "play": 1, "qa": 9, "recommendation": 73, "social": 0, "takeaway": 0, "transport": 2, "weather": 0, "__failed__": 0 }, "social": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 33, "iot": 0, "lists": 1, "music": 0, "news": 1, "play": 0, "qa": 7, "recommendation": 0, "social": 62, "takeaway": 0, "transport": 1, "weather": 0, "__failed__": 0 }, "takeaway": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 6, "iot": 2, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 4, "recommendation": 1, "social": 0, "takeaway": 44, "transport": 0, "weather": 0, "__failed__": 0 }, "transport": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 1, "email": 0, "general": 5, "iot": 0, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 4, "recommendation": 1, "social": 0, "takeaway": 1, "transport": 110, "weather": 1, "__failed__": 0 }, "weather": { "alarm": 0, "audio": 0, "calendar": 0, "cooking": 0, "datetime": 0, "email": 0, "general": 15, "iot": 3, "lists": 0, "music": 0, "news": 0, "play": 0, "qa": 3, "recommendation": 0, "social": 0, "takeaway": 0, "transport": 0, "weather": 131, "__failed__": 0 } }, "labels": [ "alarm", "audio", "calendar", "cooking", "datetime", "email", "general", "iot", "lists", "music", "news", "play", "qa", "recommendation", "social", "takeaway", "transport", "weather" ], "retrospective_majority_baseline": 0.13568521031207598 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "n": 1173, "correct": 863, "accuracy": 0.7357203751065644, "wilson95": [ 0.7097463341657818, 0.7601555339704394 ], "valid": 1173, "failures": 0, "coverage": 1.0, "heads": 1173, "head_correct": 863, "native_tie_decisions_changed": 0, "abstained_records": 0, "truncated_records": 0, "head_accuracy": 0.7357203751065644, "cluster_bootstrap95": [ 0.7075782537067545, 0.7643276696733041 ], "independent_source_groups": 47, "brier": 0.4306554195427556, "nll": 0.8712666896566365, "ece10": 0.16933220319081577, "calibration_n": 1173, "confident_wrong": 153, "macro_f1": 0.7154402352978803, "confusion_matrix": { "negative": { "negative": 138, "neutral": 32, "positive": 12, "__failed__": 0 }, "neutral": { "negative": 105, "neutral": 415, "positive": 70, "__failed__": 0 }, "positive": { "negative": 27, "neutral": 64, "positive": 310, "__failed__": 0 } }, "labels": [ "negative", "neutral", "positive" ], "retrospective_majority_baseline": 0.5029838022165388 } }, "completed_cases": 17576 } }, "paired_comparisons": { "clm-int8": { "fresh/news": { "n": 200, "local_correct": 93, "jev_correct": 178, "local_accuracy": 0.465, "jev_accuracy": 0.89, "difference_pp": -42.5, "paired_cluster_bootstrap95_pp": [ -50.5, -34.48750000000007 ], "local_only_correct": 9, "jev_only_correct": 94, "both_correct": 84, "both_wrong": 13, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 51, "jev_correct": 120, "local_accuracy": 0.255, "jev_accuracy": 0.6, "difference_pp": -34.5, "paired_cluster_bootstrap95_pp": [ -43.0, -25.0 ], "local_only_correct": 21, "jev_only_correct": 90, "both_correct": 30, "both_wrong": 59, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 132, "jev_correct": 196, "local_accuracy": 0.66, "jev_accuracy": 0.98, "difference_pp": -32.0, "paired_cluster_bootstrap95_pp": [ -39.0, -25.0 ], "local_only_correct": 2, "jev_only_correct": 66, "both_correct": 130, "both_wrong": 2, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 80, "jev_correct": 179, "local_accuracy": 0.4, "jev_accuracy": 0.895, "difference_pp": -49.5, "paired_cluster_bootstrap95_pp": [ -58.5125, -40.5 ], "local_only_correct": 20, "jev_only_correct": 119, "both_correct": 60, "both_wrong": 1, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 2, "jev_correct": 11, "local_accuracy": 0.16666666666666666, "jev_accuracy": 0.9166666666666666, "difference_pp": -75.0, "paired_cluster_bootstrap95_pp": [ -91.66666666666667, -50.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 2, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 2, "jev_correct": 10, "local_accuracy": 0.16666666666666666, "jev_accuracy": 0.8333333333333334, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -91.66666666666667, -41.666666666666664 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 2, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 6, "jev_correct": 12, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 6, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 4, "jev_correct": 12, "local_accuracy": 0.3333333333333333, "jev_accuracy": 1.0, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -91.66666666666667, -41.666666666666664 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 4, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 47, "jev_correct": 91, "local_accuracy": 0.47, "jev_accuracy": 0.91, "difference_pp": -44.0, "paired_cluster_bootstrap95_pp": [ -54.0, -33.975000000000136 ], "local_only_correct": 1, "jev_only_correct": 45, "both_correct": 46, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 22, "jev_correct": 48, "local_accuracy": 0.22, "jev_accuracy": 0.48, "difference_pp": -26.0, "paired_cluster_bootstrap95_pp": [ -36.0, -16.0 ], "local_only_correct": 5, "jev_only_correct": 31, "both_correct": 17, "both_wrong": 47, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 8, "jev_correct": 89, "local_accuracy": 0.08, "jev_accuracy": 0.89, "difference_pp": -81.0, "paired_cluster_bootstrap95_pp": [ -89.0, -72.0 ], "local_only_correct": 1, "jev_only_correct": 82, "both_correct": 7, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 33, "jev_correct": 48, "local_accuracy": 0.6875, "jev_accuracy": 1.0, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -43.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 15, "both_correct": 33, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/hard": { "n": 111, "local_correct": 38, "jev_correct": 81, "local_accuracy": 0.34234234234234234, "jev_accuracy": 0.7297297297297297, "difference_pp": -38.73873873873874, "paired_cluster_bootstrap95_pp": [ -51.351351351351354, -25.225225225225227 ], "local_only_correct": 13, "jev_only_correct": 56, "both_correct": 25, "both_wrong": 17, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 25, "jev_correct": 70, "local_accuracy": 0.3472222222222222, "jev_accuracy": 0.9722222222222222, "difference_pp": -62.5, "paired_cluster_bootstrap95_pp": [ -75.0, -48.611111111111114 ], "local_only_correct": 1, "jev_only_correct": 46, "both_correct": 24, "both_wrong": 1, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 9, "jev_correct": 18, "local_accuracy": 0.47368421052631576, "jev_accuracy": 0.9473684210526315, "difference_pp": -47.368421052631575, "paired_cluster_bootstrap95_pp": [ -73.6842105263158, -21.05263157894737 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 8, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 8, "jev_correct": 16, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 8, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 10, "jev_correct": 20, "local_accuracy": 0.45454545454545453, "jev_accuracy": 0.9090909090909091, "difference_pp": -45.45454545454545, "paired_cluster_bootstrap95_pp": [ -63.63636363636363, -22.727272727272727 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 10, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 3, "jev_correct": 16, "local_accuracy": 0.1875, "jev_accuracy": 1.0, "difference_pp": -81.25, "paired_cluster_bootstrap95_pp": [ -100.0, -62.5 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 3, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 3, "jev_correct": 18, "local_accuracy": 0.16666666666666666, "jev_accuracy": 1.0, "difference_pp": -83.33333333333334, "paired_cluster_bootstrap95_pp": [ -100.0, -66.66666666666667 ], "local_only_correct": 0, "jev_only_correct": 15, "both_correct": 3, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 8, "jev_correct": 19, "local_accuracy": 0.42105263157894735, "jev_accuracy": 1.0, "difference_pp": -57.89473684210527, "paired_cluster_bootstrap95_pp": [ -78.94736842105263, -36.8421052631579 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 8, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 7, "jev_correct": 16, "local_accuracy": 0.4117647058823529, "jev_accuracy": 0.9411764705882353, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 7, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 5, "jev_correct": 15, "local_accuracy": 0.3333333333333333, "jev_accuracy": 1.0, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -86.66666666666667, -40.0 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 5, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5, "jev_accuracy": 0.9375, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 4, "jev_correct": 18, "local_accuracy": 0.2222222222222222, "jev_accuracy": 1.0, "difference_pp": -77.77777777777779, "paired_cluster_bootstrap95_pp": [ -94.44444444444444, -55.55555555555556 ], "local_only_correct": 0, "jev_only_correct": 14, "both_correct": 4, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 7, "jev_correct": 21, "local_accuracy": 0.3181818181818182, "jev_accuracy": 0.9545454545454546, "difference_pp": -63.63636363636363, "paired_cluster_bootstrap95_pp": [ -81.81818181818181, -40.90909090909091 ], "local_only_correct": 0, "jev_only_correct": 14, "both_correct": 7, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 8, "jev_correct": 19, "local_accuracy": 0.42105263157894735, "jev_accuracy": 1.0, "difference_pp": -57.89473684210527, "paired_cluster_bootstrap95_pp": [ -78.94736842105263, -36.8421052631579 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 8, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 4, "jev_correct": 15, "local_accuracy": 0.26666666666666666, "jev_accuracy": 1.0, "difference_pp": -73.33333333333333, "paired_cluster_bootstrap95_pp": [ -93.33333333333333, -46.666666666666664 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 4, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4666666666666667, "jev_accuracy": 1.0, "difference_pp": -53.333333333333336, "paired_cluster_bootstrap95_pp": [ -80.0, -26.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 7, "jev_correct": 18, "local_accuracy": 0.3888888888888889, "jev_accuracy": 1.0, "difference_pp": -61.111111111111114, "paired_cluster_bootstrap95_pp": [ -83.33333333333333, -38.888888888888886 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 7, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.4444444444444444, "jev_accuracy": 0.9444444444444444, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -72.22222222222223, -27.77777777777778 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 6, "jev_correct": 15, "local_accuracy": 0.375, "jev_accuracy": 0.9375, "difference_pp": -56.25, "paired_cluster_bootstrap95_pp": [ -81.25, -31.25 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 6, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 7, "jev_correct": 17, "local_accuracy": 0.3888888888888889, "jev_accuracy": 0.9444444444444444, "difference_pp": -55.55555555555556, "paired_cluster_bootstrap95_pp": [ -77.77777777777777, -33.333333333333336 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 7, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 10, "jev_correct": 18, "local_accuracy": 0.5, "jev_accuracy": 0.9, "difference_pp": -40.0, "paired_cluster_bootstrap95_pp": [ -60.0, -20.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 10, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 4, "jev_correct": 18, "local_accuracy": 0.19047619047619047, "jev_accuracy": 0.8571428571428571, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -90.47619047619048, -38.095238095238095 ], "local_only_correct": 2, "jev_only_correct": 16, "both_correct": 2, "both_wrong": 1, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 7, "jev_correct": 16, "local_accuracy": 0.4375, "jev_accuracy": 1.0, "difference_pp": -56.25, "paired_cluster_bootstrap95_pp": [ -81.25, -31.25 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 7, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 9, "jev_correct": 23, "local_accuracy": 0.391304347826087, "jev_accuracy": 1.0, "difference_pp": -60.86956521739131, "paired_cluster_bootstrap95_pp": [ -82.6086956521739, -39.130434782608695 ], "local_only_correct": 0, "jev_only_correct": 14, "both_correct": 9, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 3, "jev_correct": 14, "local_accuracy": 0.21428571428571427, "jev_accuracy": 1.0, "difference_pp": -78.57142857142857, "paired_cluster_bootstrap95_pp": [ -100.0, -57.142857142857146 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 3, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 8, "jev_correct": 14, "local_accuracy": 0.5, "jev_accuracy": 0.875, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 8, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 4, "jev_correct": 14, "local_accuracy": 0.25, "jev_accuracy": 0.875, "difference_pp": -62.5, "paired_cluster_bootstrap95_pp": [ -87.5, -37.5 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 4, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 4, "jev_correct": 17, "local_accuracy": 0.23529411764705882, "jev_accuracy": 1.0, "difference_pp": -76.47058823529412, "paired_cluster_bootstrap95_pp": [ -94.11764705882354, -52.94117647058823 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 4, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 6, "jev_correct": 17, "local_accuracy": 0.35294117647058826, "jev_accuracy": 1.0, "difference_pp": -64.70588235294117, "paired_cluster_bootstrap95_pp": [ -88.23529411764706, -41.1764705882353 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 6, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.5625, "jev_accuracy": 1.0, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 5, "jev_correct": 16, "local_accuracy": 0.3125, "jev_accuracy": 1.0, "difference_pp": -68.75, "paired_cluster_bootstrap95_pp": [ -93.75, -43.75 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 5, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 5, "jev_correct": 18, "local_accuracy": 0.2777777777777778, "jev_accuracy": 1.0, "difference_pp": -72.22222222222221, "paired_cluster_bootstrap95_pp": [ -88.88888888888889, -50.0 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 5, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 4, "jev_correct": 18, "local_accuracy": 0.2222222222222222, "jev_accuracy": 1.0, "difference_pp": -77.77777777777779, "paired_cluster_bootstrap95_pp": [ -94.44444444444444, -55.55555555555556 ], "local_only_correct": 0, "jev_only_correct": 14, "both_correct": 4, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 4, "jev_correct": 17, "local_accuracy": 0.2222222222222222, "jev_accuracy": 0.9444444444444444, "difference_pp": -72.22222222222221, "paired_cluster_bootstrap95_pp": [ -88.88888888888889, -50.0 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 4, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4375, "jev_accuracy": 0.9375, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4666666666666667, "jev_accuracy": 1.0, "difference_pp": -53.333333333333336, "paired_cluster_bootstrap95_pp": [ -80.0, -26.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 4, "jev_correct": 17, "local_accuracy": 0.23529411764705882, "jev_accuracy": 1.0, "difference_pp": -76.47058823529412, "paired_cluster_bootstrap95_pp": [ -94.11764705882354, -58.8235294117647 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 4, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 7, "jev_correct": 17, "local_accuracy": 0.3684210526315789, "jev_accuracy": 0.8947368421052632, "difference_pp": -52.63157894736842, "paired_cluster_bootstrap95_pp": [ -78.94736842105263, -26.31578947368421 ], "local_only_correct": 1, "jev_only_correct": 11, "both_correct": 6, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 7, "jev_correct": 19, "local_accuracy": 0.35, "jev_accuracy": 0.95, "difference_pp": -60.0, "paired_cluster_bootstrap95_pp": [ -80.0, -40.0 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 7, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 11, "jev_correct": 20, "local_accuracy": 0.55, "jev_accuracy": 1.0, "difference_pp": -45.0, "paired_cluster_bootstrap95_pp": [ -65.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 11, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.6, "jev_accuracy": 0.8, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -45.0, 0.0 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 11, "both_wrong": 3, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 7, "jev_correct": 20, "local_accuracy": 0.3181818181818182, "jev_accuracy": 0.9090909090909091, "difference_pp": -59.09090909090909, "paired_cluster_bootstrap95_pp": [ -77.27272727272727, -40.90909090909091 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 7, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 391, "jev_correct": 629, "local_accuracy": 0.43444444444444447, "jev_accuracy": 0.6988888888888889, "difference_pp": -26.444444444444443, "paired_cluster_bootstrap95_pp": [ -35.3620547328397, -17.51908342629627 ], "local_only_correct": 88, "jev_only_correct": 326, "both_correct": 303, "both_wrong": 183, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 57, "jev_correct": 465, "local_accuracy": 0.09930313588850175, "jev_accuracy": 0.8101045296167247, "difference_pp": -71.0801393728223, "paired_cluster_bootstrap95_pp": [ -74.91289198606272, -67.24738675958189 ], "local_only_correct": 4, "jev_only_correct": 412, "both_correct": 53, "both_wrong": 105, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 134, "jev_correct": 490, "local_accuracy": 0.19142857142857142, "jev_accuracy": 0.7, "difference_pp": -50.857142857142854, "paired_cluster_bootstrap95_pp": [ -55.49174295235969, -46.4837807126186 ], "local_only_correct": 34, "jev_only_correct": 390, "both_correct": 100, "both_wrong": 176, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 5, "jev_correct": 97, "local_accuracy": 0.0125, "jev_accuracy": 0.2425, "difference_pp": -23.0, "paired_cluster_bootstrap95_pp": [ -27.25, -18.75 ], "local_only_correct": 5, "jev_only_correct": 97, "both_correct": 0, "both_wrong": 298, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 15, "jev_correct": 28, "local_accuracy": 0.5357142857142857, "jev_accuracy": 1.0, "difference_pp": -46.42857142857143, "paired_cluster_bootstrap95_pp": [ -64.28571428571429, -28.571428571428573 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 15, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 15, "jev_correct": 28, "local_accuracy": 0.5357142857142857, "jev_accuracy": 1.0, "difference_pp": -46.42857142857143, "paired_cluster_bootstrap95_pp": [ -64.28571428571429, -28.571428571428573 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 15, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 8, "jev_correct": 28, "local_accuracy": 0.2857142857142857, "jev_accuracy": 1.0, "difference_pp": -71.42857142857143, "paired_cluster_bootstrap95_pp": [ -85.71428571428571, -53.57142857142857 ], "local_only_correct": 0, "jev_only_correct": 20, "both_correct": 8, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 8, "jev_correct": 28, "local_accuracy": 0.2857142857142857, "jev_accuracy": 1.0, "difference_pp": -71.42857142857143, "paired_cluster_bootstrap95_pp": [ -85.71428571428571, -53.57142857142857 ], "local_only_correct": 0, "jev_only_correct": 20, "both_correct": 8, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 8, "jev_correct": 40, "local_accuracy": 0.2, "jev_accuracy": 1.0, "difference_pp": -80.0, "paired_cluster_bootstrap95_pp": [ -90.0, -67.5 ], "local_only_correct": 0, "jev_only_correct": 32, "both_correct": 8, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 8, "jev_correct": 40, "local_accuracy": 0.2, "jev_accuracy": 1.0, "difference_pp": -80.0, "paired_cluster_bootstrap95_pp": [ -90.0, -67.5 ], "local_only_correct": 0, "jev_only_correct": 32, "both_correct": 8, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 13, "jev_correct": 39, "local_accuracy": 0.325, "jev_accuracy": 0.975, "difference_pp": -65.0, "paired_cluster_bootstrap95_pp": [ -80.0, -50.0 ], "local_only_correct": 0, "jev_only_correct": 26, "both_correct": 13, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 13, "jev_correct": 39, "local_accuracy": 0.325, "jev_accuracy": 0.975, "difference_pp": -65.0, "paired_cluster_bootstrap95_pp": [ -80.0, -50.0 ], "local_only_correct": 0, "jev_only_correct": 26, "both_correct": 13, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 99, "jev_correct": 237, "local_accuracy": 0.2475, "jev_accuracy": 0.5925, "difference_pp": -34.5, "paired_cluster_bootstrap95_pp": [ -40.0, -28.75 ], "local_only_correct": 32, "jev_only_correct": 170, "both_correct": 67, "both_wrong": 131, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 90, "jev_correct": 277, "local_accuracy": 0.3, "jev_accuracy": 0.9233333333333333, "difference_pp": -62.33333333333333, "paired_cluster_bootstrap95_pp": [ -68.0, -56.666666666666664 ], "local_only_correct": 2, "jev_only_correct": 189, "both_correct": 88, "both_wrong": 21, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 329, "jev_correct": 377, "local_accuracy": 0.8225, "jev_accuracy": 0.9425, "difference_pp": -12.0, "paired_cluster_bootstrap95_pp": [ -16.25, -7.75 ], "local_only_correct": 17, "jev_only_correct": 65, "both_correct": 312, "both_wrong": 6, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 155, "jev_correct": 346, "local_accuracy": 0.3875, "jev_accuracy": 0.865, "difference_pp": -47.75, "paired_cluster_bootstrap95_pp": [ -53.5, -41.75 ], "local_only_correct": 22, "jev_only_correct": 213, "both_correct": 133, "both_wrong": 32, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 169, "jev_correct": 360, "local_accuracy": 0.4225, "jev_accuracy": 0.9, "difference_pp": -47.75, "paired_cluster_bootstrap95_pp": [ -54.5, -41.25 ], "local_only_correct": 39, "jev_only_correct": 230, "both_correct": 130, "both_wrong": 1, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 11, "jev_correct": 225, "local_accuracy": 0.03666666666666667, "jev_accuracy": 0.75, "difference_pp": -71.33333333333334, "paired_cluster_bootstrap95_pp": [ -76.66666666666667, -66.0 ], "local_only_correct": 0, "jev_only_correct": 214, "both_correct": 11, "both_wrong": 75, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 274, "jev_correct": 393, "local_accuracy": 0.685, "jev_accuracy": 0.9825, "difference_pp": -29.75, "paired_cluster_bootstrap95_pp": [ -34.5, -25.0 ], "local_only_correct": 4, "jev_only_correct": 123, "both_correct": 270, "both_wrong": 3, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 200, "jev_correct": 265, "local_accuracy": 0.5, "jev_accuracy": 0.6625, "difference_pp": -16.25, "paired_cluster_bootstrap95_pp": [ -20.25625, -12.75 ], "local_only_correct": 7, "jev_only_correct": 72, "both_correct": 193, "both_wrong": 128, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 54, "jev_correct": 75, "local_accuracy": 0.54, "jev_accuracy": 0.75, "difference_pp": -21.0, "paired_cluster_bootstrap95_pp": [ -30.0, -12.0 ], "local_only_correct": 2, "jev_only_correct": 23, "both_correct": 52, "both_wrong": 23, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 6, "jev_correct": 56, "local_accuracy": 0.06, "jev_accuracy": 0.56, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -60.0, -40.0 ], "local_only_correct": 1, "jev_only_correct": 51, "both_correct": 5, "both_wrong": 43, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 16, "jev_correct": 49, "local_accuracy": 0.16, "jev_accuracy": 0.49, "difference_pp": -33.0, "paired_cluster_bootstrap95_pp": [ -43.0, -23.0 ], "local_only_correct": 1, "jev_only_correct": 34, "both_correct": 15, "both_wrong": 50, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 3, "jev_correct": 48, "local_accuracy": 0.03, "jev_accuracy": 0.48, "difference_pp": -45.0, "paired_cluster_bootstrap95_pp": [ -56.0, -35.0 ], "local_only_correct": 1, "jev_only_correct": 46, "both_correct": 2, "both_wrong": 51, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 15, "jev_correct": 79, "local_accuracy": 0.15, "jev_accuracy": 0.79, "difference_pp": -64.0, "paired_cluster_bootstrap95_pp": [ -74.0, -53.0 ], "local_only_correct": 3, "jev_only_correct": 67, "both_correct": 12, "both_wrong": 18, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 0, "jev_correct": 6, "local_accuracy": 0.0, "jev_accuracy": 0.06, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -11.0, -2.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 0, "both_wrong": 94, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 21, "jev_correct": 64, "local_accuracy": 0.21, "jev_accuracy": 0.64, "difference_pp": -43.0, "paired_cluster_bootstrap95_pp": [ -54.0, -31.975000000000136 ], "local_only_correct": 4, "jev_only_correct": 47, "both_correct": 17, "both_wrong": 32, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 13, "jev_correct": 37, "local_accuracy": 0.13, "jev_accuracy": 0.37, "difference_pp": -24.0, "paired_cluster_bootstrap95_pp": [ -36.0, -12.975000000000136 ], "local_only_correct": 9, "jev_only_correct": 33, "both_correct": 4, "both_wrong": 54, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 0, "jev_correct": 16, "local_accuracy": 0.0, "jev_accuracy": 0.16, "difference_pp": -16.0, "paired_cluster_bootstrap95_pp": [ -24.0, -9.0 ], "local_only_correct": 0, "jev_only_correct": 16, "both_correct": 0, "both_wrong": 84, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 0, "jev_correct": 34, "local_accuracy": 0.0, "jev_accuracy": 0.34, "difference_pp": -34.0, "paired_cluster_bootstrap95_pp": [ -43.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 34, "both_correct": 0, "both_wrong": 66, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 31, "jev_correct": 83, "local_accuracy": 0.31, "jev_accuracy": 0.83, "difference_pp": -52.0, "paired_cluster_bootstrap95_pp": [ -63.0, -40.0 ], "local_only_correct": 4, "jev_only_correct": 56, "both_correct": 27, "both_wrong": 13, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 3, "jev_correct": 21, "local_accuracy": 0.03, "jev_accuracy": 0.21, "difference_pp": -18.0, "paired_cluster_bootstrap95_pp": [ -27.0, -10.0 ], "local_only_correct": 2, "jev_only_correct": 20, "both_correct": 1, "both_wrong": 77, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 5, "jev_correct": 51, "local_accuracy": 0.05, "jev_accuracy": 0.51, "difference_pp": -46.0, "paired_cluster_bootstrap95_pp": [ -57.0, -35.0 ], "local_only_correct": 3, "jev_only_correct": 49, "both_correct": 2, "both_wrong": 46, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 4, "jev_correct": 65, "local_accuracy": 0.04, "jev_accuracy": 0.65, "difference_pp": -61.0, "paired_cluster_bootstrap95_pp": [ -72.0, -50.0 ], "local_only_correct": 3, "jev_only_correct": 64, "both_correct": 1, "both_wrong": 32, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 8, "jev_correct": 39, "local_accuracy": 0.08, "jev_accuracy": 0.39, "difference_pp": -31.0, "paired_cluster_bootstrap95_pp": [ -42.0, -20.0 ], "local_only_correct": 5, "jev_only_correct": 36, "both_correct": 3, "both_wrong": 56, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 0, "jev_correct": 9, "local_accuracy": 0.0, "jev_accuracy": 0.09, "difference_pp": -9.0, "paired_cluster_bootstrap95_pp": [ -15.0, -4.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 0, "both_wrong": 91, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 5, "jev_correct": 59, "local_accuracy": 0.05, "jev_accuracy": 0.59, "difference_pp": -54.0, "paired_cluster_bootstrap95_pp": [ -64.0, -44.0 ], "local_only_correct": 1, "jev_only_correct": 55, "both_correct": 4, "both_wrong": 40, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 23, "jev_correct": 91, "local_accuracy": 0.23, "jev_accuracy": 0.91, "difference_pp": -68.0, "paired_cluster_bootstrap95_pp": [ -79.0, -56.0 ], "local_only_correct": 7, "jev_only_correct": 75, "both_correct": 16, "both_wrong": 2, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 3, "jev_correct": 88, "local_accuracy": 0.03, "jev_accuracy": 0.88, "difference_pp": -85.0, "paired_cluster_bootstrap95_pp": [ -92.0, -78.0 ], "local_only_correct": 0, "jev_only_correct": 85, "both_correct": 3, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 21, "jev_correct": 48, "local_accuracy": 0.21, "jev_accuracy": 0.48, "difference_pp": -27.0, "paired_cluster_bootstrap95_pp": [ -40.0, -14.0 ], "local_only_correct": 11, "jev_only_correct": 38, "both_correct": 10, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 1229, "jev_correct": 2550, "local_accuracy": 0.4168928086838535, "jev_accuracy": 0.8649932157394844, "difference_pp": -44.81004070556309, "paired_cluster_bootstrap95_pp": [ -46.70963364993216, -42.73999321573949 ], "local_only_correct": 64, "jev_only_correct": 1385, "both_correct": 1165, "both_wrong": 334, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 894, "jev_correct": 2412, "local_accuracy": 0.30325644504748983, "jev_accuracy": 0.8181818181818182, "difference_pp": -51.49253731343284, "paired_cluster_bootstrap95_pp": [ -53.493894165535956, -49.423337856173674 ], "local_only_correct": 88, "jev_only_correct": 1606, "both_correct": 806, "both_wrong": 448, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 522, "jev_correct": 843, "local_accuracy": 0.44501278772378516, "jev_accuracy": 0.7186700767263428, "difference_pp": -27.365728900255753, "paired_cluster_bootstrap95_pp": [ -33.72989561637274, -21.324627947645013 ], "local_only_correct": 207, "jev_only_correct": 528, "both_correct": 315, "both_wrong": 123, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "cygnet-12b-nf4": { "fresh/news": { "n": 200, "local_correct": 169, "jev_correct": 178, "local_accuracy": 0.845, "jev_accuracy": 0.89, "difference_pp": -4.5, "paired_cluster_bootstrap95_pp": [ -8.5, -1.0 ], "local_only_correct": 3, "jev_only_correct": 12, "both_correct": 166, "both_wrong": 19, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 49, "jev_correct": 120, "local_accuracy": 0.245, "jev_accuracy": 0.6, "difference_pp": -35.5, "paired_cluster_bootstrap95_pp": [ -44.0, -26.5 ], "local_only_correct": 18, "jev_only_correct": 89, "both_correct": 31, "both_wrong": 62, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 187, "jev_correct": 196, "local_accuracy": 0.935, "jev_accuracy": 0.98, "difference_pp": -4.5, "paired_cluster_bootstrap95_pp": [ -8.0, -1.5 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 186, "both_wrong": 3, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 178, "jev_correct": 179, "local_accuracy": 0.89, "jev_accuracy": 0.895, "difference_pp": -0.5, "paired_cluster_bootstrap95_pp": [ -5.0, 4.0 ], "local_only_correct": 9, "jev_only_correct": 10, "both_correct": 169, "both_wrong": 12, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 10, "jev_correct": 11, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9166666666666666, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 10, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 10, "jev_correct": 10, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.8333333333333334, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 10, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 88, "jev_correct": 91, "local_accuracy": 0.88, "jev_accuracy": 0.91, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -7.024999999999999, 1.0 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 87, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 43, "jev_correct": 48, "local_accuracy": 0.43, "jev_accuracy": 0.48, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -13.0, 3.0 ], "local_only_correct": 6, "jev_only_correct": 11, "both_correct": 37, "both_wrong": 46, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 0, "jev_correct": 89, "local_accuracy": 0.0, "jev_accuracy": 0.89, "difference_pp": -89.0, "paired_cluster_bootstrap95_pp": [ -95.0, -82.0 ], "local_only_correct": 0, "jev_only_correct": 89, "both_correct": 0, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 82, "jev_correct": 81, "local_accuracy": 0.7387387387387387, "jev_accuracy": 0.7297297297297297, "difference_pp": 0.9009009009009009, "paired_cluster_bootstrap95_pp": [ -8.108108108108109, 9.90990990990991 ], "local_only_correct": 13, "jev_only_correct": 12, "both_correct": 69, "both_wrong": 17, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 65, "jev_correct": 70, "local_accuracy": 0.9027777777777778, "jev_accuracy": 0.9722222222222222, "difference_pp": -6.944444444444445, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 65, "both_wrong": 2, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 19, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 0.9473684210526315, "difference_pp": 5.263157894736842, "paired_cluster_bootstrap95_pp": [ 0.0, 15.789473684210526 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 13.636363636363637 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.8235294117647058, "jev_accuracy": 0.9411764705882353, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/content_type": { "n": 22, "local_correct": 21, "jev_correct": 21, "local_accuracy": 0.9545454545454546, "jev_accuracy": 0.9545454545454546, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 21, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/expense_category": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 18, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 0.9444444444444444, "difference_pp": 5.555555555555555, "paired_cluster_bootstrap95_pp": [ 0.0, 16.666666666666668 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -25.0, 12.5 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 18, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 0.9444444444444444, "difference_pp": 5.555555555555555, "paired_cluster_bootstrap95_pp": [ 0.0, 16.666666666666668 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.9, "jev_accuracy": 0.9, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.8571428571428571, "jev_accuracy": 0.8571428571428571, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -23.80952380952381, 23.80952380952381 ], "local_only_correct": 3, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 13, "jev_correct": 14, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -21.428571428571427, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 13, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 13, "jev_correct": 14, "local_accuracy": 0.8125, "jev_accuracy": 0.875, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -25.0, 12.5 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 13, "jev_correct": 14, "local_accuracy": 0.8125, "jev_accuracy": 0.875, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -25.0, 12.5 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.8125, "jev_accuracy": 1.0, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7222222222222222, "jev_accuracy": 0.9444444444444444, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8421052631578947, "jev_accuracy": 0.8947368421052632, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 2, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9, "jev_accuracy": 0.95, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -20.0, 10.0 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 17, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 20, "jev_correct": 20, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.7, "jev_accuracy": 0.8, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -30.0, 10.0 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 3, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 21, "jev_correct": 20, "local_accuracy": 0.9545454545454546, "jev_accuracy": 0.9090909090909091, "difference_pp": 4.545454545454546, "paired_cluster_bootstrap95_pp": [ -9.090909090909092, 22.727272727272727 ], "local_only_correct": 2, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 602, "jev_correct": 629, "local_accuracy": 0.6688888888888889, "jev_accuracy": 0.6988888888888889, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -5.64014136283998, -0.3165941120368196 ], "local_only_correct": 61, "jev_only_correct": 88, "both_correct": 541, "both_wrong": 210, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 449, "jev_correct": 465, "local_accuracy": 0.7822299651567944, "jev_accuracy": 0.8101045296167247, "difference_pp": -2.7874564459930316, "paired_cluster_bootstrap95_pp": [ -6.10191637630662, 0.34843205574912894 ], "local_only_correct": 38, "jev_only_correct": 54, "both_correct": 411, "both_wrong": 71, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 288, "jev_correct": 490, "local_accuracy": 0.4114285714285714, "jev_accuracy": 0.7, "difference_pp": -28.857142857142858, "paired_cluster_bootstrap95_pp": [ -33.47581417624521, -24.327288055682086 ], "local_only_correct": 55, "jev_only_correct": 257, "both_correct": 233, "both_wrong": 155, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 79, "jev_correct": 97, "local_accuracy": 0.1975, "jev_accuracy": 0.2425, "difference_pp": -4.5, "paired_cluster_bootstrap95_pp": [ -9.25, 0.25 ], "local_only_correct": 40, "jev_only_correct": 58, "both_correct": 39, "both_wrong": 263, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en/reversed": { "n": 40, "local_correct": 40, "jev_correct": 40, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 40, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en": { "n": 40, "local_correct": 40, "jev_correct": 40, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 40, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 40, "jev_correct": 39, "local_accuracy": 1.0, "jev_accuracy": 0.975, "difference_pp": 2.5, "paired_cluster_bootstrap95_pp": [ 0.0, 7.5 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 39, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 38, "jev_correct": 39, "local_accuracy": 0.95, "jev_accuracy": 0.975, "difference_pp": -2.5, "paired_cluster_bootstrap95_pp": [ -12.5, 5.0 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 37, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 107, "jev_correct": 237, "local_accuracy": 0.2675, "jev_accuracy": 0.5925, "difference_pp": -32.5, "paired_cluster_bootstrap95_pp": [ -38.25, -26.5 ], "local_only_correct": 31, "jev_only_correct": 161, "both_correct": 76, "both_wrong": 132, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 271, "jev_correct": 277, "local_accuracy": 0.9033333333333333, "jev_accuracy": 0.9233333333333333, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -4.666666666666667, 1.0 ], "local_only_correct": 6, "jev_only_correct": 12, "both_correct": 265, "both_wrong": 17, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 371, "jev_correct": 377, "local_accuracy": 0.9275, "jev_accuracy": 0.9425, "difference_pp": -1.5, "paired_cluster_bootstrap95_pp": [ -3.5, 0.5 ], "local_only_correct": 6, "jev_only_correct": 12, "both_correct": 365, "both_wrong": 17, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 338, "jev_correct": 346, "local_accuracy": 0.845, "jev_accuracy": 0.865, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -4.0, 0.0 ], "local_only_correct": 5, "jev_only_correct": 13, "both_correct": 333, "both_wrong": 49, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 362, "jev_correct": 360, "local_accuracy": 0.905, "jev_accuracy": 0.9, "difference_pp": 0.5, "paired_cluster_bootstrap95_pp": [ -2.25, 3.5 ], "local_only_correct": 18, "jev_only_correct": 16, "both_correct": 344, "both_wrong": 22, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 186, "jev_correct": 225, "local_accuracy": 0.62, "jev_accuracy": 0.75, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -17.333333333333332, -8.666666666666666 ], "local_only_correct": 7, "jev_only_correct": 46, "both_correct": 179, "both_wrong": 68, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 380, "jev_correct": 393, "local_accuracy": 0.95, "jev_accuracy": 0.9825, "difference_pp": -3.25, "paired_cluster_bootstrap95_pp": [ -5.25, -1.5 ], "local_only_correct": 1, "jev_only_correct": 14, "both_correct": 379, "both_wrong": 6, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 292, "jev_correct": 265, "local_accuracy": 0.73, "jev_accuracy": 0.6625, "difference_pp": 6.75, "paired_cluster_bootstrap95_pp": [ 3.2437500000000004, 10.5 ], "local_only_correct": 42, "jev_only_correct": 15, "both_correct": 250, "both_wrong": 93, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 77, "jev_correct": 75, "local_accuracy": 0.77, "jev_accuracy": 0.75, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -4.0, 8.0 ], "local_only_correct": 6, "jev_only_correct": 4, "both_correct": 71, "both_wrong": 19, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 45, "jev_correct": 56, "local_accuracy": 0.45, "jev_accuracy": 0.56, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -19.0, -3.0 ], "local_only_correct": 3, "jev_only_correct": 14, "both_correct": 42, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 44, "jev_correct": 49, "local_accuracy": 0.44, "jev_accuracy": 0.49, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -13.0, 4.0 ], "local_only_correct": 7, "jev_only_correct": 12, "both_correct": 37, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 35, "jev_correct": 48, "local_accuracy": 0.35, "jev_accuracy": 0.48, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -23.0, -3.0 ], "local_only_correct": 7, "jev_only_correct": 20, "both_correct": 28, "both_wrong": 45, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 76, "jev_correct": 79, "local_accuracy": 0.76, "jev_accuracy": 0.79, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -8.0, 1.0 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 75, "both_wrong": 20, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 8, "jev_correct": 6, "local_accuracy": 0.08, "jev_accuracy": 0.06, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -3.0, 7.0 ], "local_only_correct": 4, "jev_only_correct": 2, "both_correct": 4, "both_wrong": 90, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 61, "jev_correct": 64, "local_accuracy": 0.61, "jev_accuracy": 0.64, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -9.0, 3.0 ], "local_only_correct": 3, "jev_only_correct": 6, "both_correct": 58, "both_wrong": 33, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 37, "jev_correct": 37, "local_accuracy": 0.37, "jev_accuracy": 0.37, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -6.0, 5.0 ], "local_only_correct": 4, "jev_only_correct": 4, "both_correct": 33, "both_wrong": 59, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 17, "jev_correct": 16, "local_accuracy": 0.17, "jev_accuracy": 0.16, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 7.0 ], "local_only_correct": 5, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 79, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 29, "jev_correct": 34, "local_accuracy": 0.29, "jev_accuracy": 0.34, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -13.0, 3.0 ], "local_only_correct": 6, "jev_only_correct": 11, "both_correct": 23, "both_wrong": 60, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 87, "jev_correct": 83, "local_accuracy": 0.87, "jev_accuracy": 0.83, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -2.0, 10.0 ], "local_only_correct": 7, "jev_only_correct": 3, "both_correct": 80, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 23, "jev_correct": 21, "local_accuracy": 0.23, "jev_accuracy": 0.21, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -6.0, 10.0 ], "local_only_correct": 9, "jev_only_correct": 7, "both_correct": 14, "both_wrong": 70, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 51, "jev_correct": 51, "local_accuracy": 0.51, "jev_accuracy": 0.51, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -8.0, 8.0 ], "local_only_correct": 8, "jev_only_correct": 8, "both_correct": 43, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 0, "jev_correct": 65, "local_accuracy": 0.0, "jev_accuracy": 0.65, "difference_pp": -65.0, "paired_cluster_bootstrap95_pp": [ -75.0, -56.0 ], "local_only_correct": 0, "jev_only_correct": 65, "both_correct": 0, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 33, "jev_correct": 39, "local_accuracy": 0.33, "jev_accuracy": 0.39, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -11.0, -1.0 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 32, "both_wrong": 60, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 7, "jev_correct": 9, "local_accuracy": 0.07, "jev_accuracy": 0.09, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -7.0, 3.0 ], "local_only_correct": 3, "jev_only_correct": 5, "both_correct": 4, "both_wrong": 88, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 59, "jev_correct": 59, "local_accuracy": 0.59, "jev_accuracy": 0.59, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -6.0, 7.0 ], "local_only_correct": 5, "jev_only_correct": 5, "both_correct": 54, "both_wrong": 36, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 88, "jev_correct": 91, "local_accuracy": 0.88, "jev_accuracy": 0.91, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -8.0, 2.0 ], "local_only_correct": 2, "jev_only_correct": 5, "both_correct": 86, "both_wrong": 7, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 0, "jev_correct": 88, "local_accuracy": 0.0, "jev_accuracy": 0.88, "difference_pp": -88.0, "paired_cluster_bootstrap95_pp": [ -94.0, -81.0 ], "local_only_correct": 0, "jev_only_correct": 88, "both_correct": 0, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 45, "jev_correct": 48, "local_accuracy": 0.45, "jev_accuracy": 0.48, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -10.0, 4.0 ], "local_only_correct": 5, "jev_only_correct": 8, "both_correct": 40, "both_wrong": 47, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2272, "jev_correct": 2550, "local_accuracy": 0.7706919945725916, "jev_accuracy": 0.8649932157394844, "difference_pp": -9.430122116689281, "paired_cluster_bootstrap95_pp": [ -10.888738127544098, -8.039348710990502 ], "local_only_correct": 95, "jev_only_correct": 373, "both_correct": 2177, "both_wrong": 303, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2013, "jev_correct": 2412, "local_accuracy": 0.6828358208955224, "jev_accuracy": 0.8181818181818182, "difference_pp": -13.534599728629578, "paired_cluster_bootstrap95_pp": [ -15.162822252374491, -11.906377204884668 ], "local_only_correct": 120, "jev_only_correct": 519, "both_correct": 1893, "both_wrong": 416, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 758, "jev_correct": 843, "local_accuracy": 0.6462063086104007, "jev_accuracy": 0.7186700767263428, "difference_pp": -7.246376811594203, "paired_cluster_bootstrap95_pp": [ -10.050519365118275, -4.333782779767034 ], "local_only_correct": 73, "jev_only_correct": 158, "both_correct": 685, "both_wrong": 257, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "decider-08b": { "fresh/news": { "n": 200, "local_correct": 174, "jev_correct": 178, "local_accuracy": 0.87, "jev_accuracy": 0.89, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -6.5, 2.5 ], "local_only_correct": 8, "jev_only_correct": 12, "both_correct": 166, "both_wrong": 14, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 150, "jev_correct": 120, "local_accuracy": 0.75, "jev_accuracy": 0.6, "difference_pp": 15.0, "paired_cluster_bootstrap95_pp": [ 9.0, 21.5 ], "local_only_correct": 38, "jev_only_correct": 8, "both_correct": 112, "both_wrong": 42, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 196, "jev_correct": 196, "local_accuracy": 0.98, "jev_accuracy": 0.98, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -1.5, 1.5 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 195, "both_wrong": 3, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 184, "jev_correct": 179, "local_accuracy": 0.92, "jev_accuracy": 0.895, "difference_pp": 2.5, "paired_cluster_bootstrap95_pp": [ -1.5, 6.5 ], "local_only_correct": 11, "jev_only_correct": 6, "both_correct": 173, "both_wrong": 10, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 6, "jev_correct": 11, "local_accuracy": 0.5, "jev_accuracy": 0.9166666666666666, "difference_pp": -41.66666666666667, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 6, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 5, "jev_correct": 10, "local_accuracy": 0.4166666666666667, "jev_accuracy": 0.8333333333333334, "difference_pp": -41.66666666666667, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 5, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 11, "jev_correct": 12, "local_accuracy": 0.9166666666666666, "jev_accuracy": 1.0, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 11, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 81, "jev_correct": 91, "local_accuracy": 0.81, "jev_accuracy": 0.91, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -16.0, -4.0 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 81, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 61, "jev_correct": 48, "local_accuracy": 0.61, "jev_accuracy": 0.48, "difference_pp": 13.0, "paired_cluster_bootstrap95_pp": [ 2.0, 24.0 ], "local_only_correct": 23, "jev_only_correct": 10, "both_correct": 38, "both_wrong": 29, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 78, "jev_correct": 89, "local_accuracy": 0.78, "jev_accuracy": 0.89, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -20.0, -2.0 ], "local_only_correct": 5, "jev_only_correct": 16, "both_correct": 73, "both_wrong": 6, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 43, "jev_correct": 81, "local_accuracy": 0.38738738738738737, "jev_accuracy": 0.7297297297297297, "difference_pp": -34.234234234234236, "paired_cluster_bootstrap95_pp": [ -45.04504504504504, -23.423423423423422 ], "local_only_correct": 6, "jev_only_correct": 44, "both_correct": 37, "both_wrong": 24, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 61, "jev_correct": 70, "local_accuracy": 0.8472222222222222, "jev_accuracy": 0.9722222222222222, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -22.22222222222222, -4.131944444444634 ], "local_only_correct": 2, "jev_only_correct": 11, "both_correct": 59, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 11, "jev_correct": 18, "local_accuracy": 0.5789473684210527, "jev_accuracy": 0.9473684210526315, "difference_pp": -36.84210526315789, "paired_cluster_bootstrap95_pp": [ -57.89473684210526, -15.789473684210526 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 11, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 10, "jev_correct": 16, "local_accuracy": 0.625, "jev_accuracy": 1.0, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 10, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 11, "jev_correct": 20, "local_accuracy": 0.5, "jev_accuracy": 0.9090909090909091, "difference_pp": -40.909090909090914, "paired_cluster_bootstrap95_pp": [ -68.18181818181819, -13.636363636363637 ], "local_only_correct": 2, "jev_only_correct": 11, "both_correct": 9, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -38.888888888888886, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.7647058823529411, "jev_accuracy": 0.9411764705882353, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.7222222222222222, "jev_accuracy": 1.0, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 15, "jev_correct": 21, "local_accuracy": 0.6818181818181818, "jev_accuracy": 0.9545454545454546, "difference_pp": -27.27272727272727, "paired_cluster_bootstrap95_pp": [ -45.45454545454545, -9.090909090909092 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 15, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7222222222222222, "jev_accuracy": 0.9444444444444444, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, 0.0 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 5, "jev_correct": 15, "local_accuracy": 0.3125, "jev_accuracy": 0.9375, "difference_pp": -62.5, "paired_cluster_bootstrap95_pp": [ -87.5, -37.5 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 5, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7777777777777778, "jev_accuracy": 0.9444444444444444, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7, "jev_accuracy": 0.9, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, -5.0 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 5, "jev_correct": 18, "local_accuracy": 0.23809523809523808, "jev_accuracy": 0.8571428571428571, "difference_pp": -61.904761904761905, "paired_cluster_bootstrap95_pp": [ -80.95238095238095, -38.095238095238095 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 5, "both_wrong": 3, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.5625, "jev_accuracy": 1.0, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 8, "jev_correct": 14, "local_accuracy": 0.5714285714285714, "jev_accuracy": 1.0, "difference_pp": -42.857142857142854, "paired_cluster_bootstrap95_pp": [ -71.42857142857143, -21.428571428571427 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 8, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 9, "jev_correct": 14, "local_accuracy": 0.5625, "jev_accuracy": 0.875, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 9, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.6875, "jev_accuracy": 0.875, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -50.0, 12.5 ], "local_only_correct": 2, "jev_only_correct": 5, "both_correct": 9, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.8125, "jev_accuracy": 1.0, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.8125, "jev_accuracy": 1.0, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7777777777777778, "jev_accuracy": 1.0, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9444444444444444, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 6, "jev_correct": 15, "local_accuracy": 0.375, "jev_accuracy": 0.9375, "difference_pp": -56.25, "paired_cluster_bootstrap95_pp": [ -81.25, -31.25 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 6, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -53.333333333333336, -13.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.5789473684210527, "jev_accuracy": 0.8947368421052632, "difference_pp": -31.57894736842105, "paired_cluster_bootstrap95_pp": [ -57.89473684210526, -5.2631578947368425 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 10, "jev_correct": 19, "local_accuracy": 0.5, "jev_accuracy": 0.95, "difference_pp": -45.0, "paired_cluster_bootstrap95_pp": [ -65.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 10, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.6, "jev_accuracy": 0.8, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -50.0, 15.0 ], "local_only_correct": 4, "jev_only_correct": 8, "both_correct": 8, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9090909090909091, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -22.727272727272727, 9.090909090909092 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 18, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 431, "jev_correct": 629, "local_accuracy": 0.47888888888888886, "jev_accuracy": 0.6988888888888889, "difference_pp": -22.0, "paired_cluster_bootstrap95_pp": [ -26.94288076211471, -17.69322230484922 ], "local_only_correct": 68, "jev_only_correct": 266, "both_correct": 363, "both_wrong": 203, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 320, "jev_correct": 465, "local_accuracy": 0.5574912891986062, "jev_accuracy": 0.8101045296167247, "difference_pp": -25.261324041811843, "paired_cluster_bootstrap95_pp": [ -29.442508710801395, -21.254355400696863 ], "local_only_correct": 20, "jev_only_correct": 165, "both_correct": 300, "both_wrong": 89, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 178, "jev_correct": 490, "local_accuracy": 0.2542857142857143, "jev_accuracy": 0.7, "difference_pp": -44.57142857142857, "paired_cluster_bootstrap95_pp": [ -48.77732636081746, -40.11174785100288 ], "local_only_correct": 35, "jev_only_correct": 347, "both_correct": 143, "both_wrong": 175, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 20, "jev_correct": 97, "local_accuracy": 0.05, "jev_accuracy": 0.2425, "difference_pp": -19.25, "paired_cluster_bootstrap95_pp": [ -24.0, -14.75 ], "local_only_correct": 14, "jev_only_correct": 91, "both_correct": 6, "both_wrong": 289, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 26, "jev_correct": 28, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -17.857142857142858, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 26, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 26, "jev_correct": 28, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -17.857142857142858, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 26, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 25, "jev_correct": 28, "local_accuracy": 0.8928571428571429, "jev_accuracy": 1.0, "difference_pp": -10.714285714285714, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 25, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 34, "jev_correct": 40, "local_accuracy": 0.85, "jev_accuracy": 1.0, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -27.5, -5.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 34, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 35, "jev_correct": 40, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -25.0, -2.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 35, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 28, "jev_correct": 39, "local_accuracy": 0.7, "jev_accuracy": 0.975, "difference_pp": -27.500000000000004, "paired_cluster_bootstrap95_pp": [ -42.5, -12.5 ], "local_only_correct": 1, "jev_only_correct": 12, "both_correct": 27, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 30, "jev_correct": 39, "local_accuracy": 0.75, "jev_accuracy": 0.975, "difference_pp": -22.5, "paired_cluster_bootstrap95_pp": [ -35.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 30, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 321, "jev_correct": 237, "local_accuracy": 0.8025, "jev_accuracy": 0.5925, "difference_pp": 21.0, "paired_cluster_bootstrap95_pp": [ 16.5, 25.75 ], "local_only_correct": 95, "jev_only_correct": 11, "both_correct": 226, "both_wrong": 68, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 275, "jev_correct": 277, "local_accuracy": 0.9166666666666666, "jev_accuracy": 0.9233333333333333, "difference_pp": -0.6666666666666667, "paired_cluster_bootstrap95_pp": [ -3.6666666666666665, 2.6666666666666665 ], "local_only_correct": 12, "jev_only_correct": 14, "both_correct": 263, "both_wrong": 11, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 367, "jev_correct": 377, "local_accuracy": 0.9175, "jev_accuracy": 0.9425, "difference_pp": -2.5, "paired_cluster_bootstrap95_pp": [ -6.0, 0.75 ], "local_only_correct": 17, "jev_only_correct": 27, "both_correct": 350, "both_wrong": 6, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 362, "jev_correct": 346, "local_accuracy": 0.905, "jev_accuracy": 0.865, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ 1.25, 7.0 ], "local_only_correct": 26, "jev_only_correct": 10, "both_correct": 336, "both_wrong": 28, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 370, "jev_correct": 360, "local_accuracy": 0.925, "jev_accuracy": 0.9, "difference_pp": 2.5, "paired_cluster_bootstrap95_pp": [ -0.25, 5.5 ], "local_only_correct": 22, "jev_only_correct": 12, "both_correct": 348, "both_wrong": 18, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 205, "jev_correct": 225, "local_accuracy": 0.6833333333333333, "jev_accuracy": 0.75, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -10.0, -3.0 ], "local_only_correct": 5, "jev_only_correct": 25, "both_correct": 200, "both_wrong": 70, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 388, "jev_correct": 393, "local_accuracy": 0.97, "jev_accuracy": 0.9825, "difference_pp": -1.25, "paired_cluster_bootstrap95_pp": [ -2.75, 0.25 ], "local_only_correct": 2, "jev_only_correct": 7, "both_correct": 386, "both_wrong": 5, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 297, "jev_correct": 265, "local_accuracy": 0.7425, "jev_accuracy": 0.6625, "difference_pp": 8.0, "paired_cluster_bootstrap95_pp": [ 3.25, 12.5 ], "local_only_correct": 64, "jev_only_correct": 32, "both_correct": 233, "both_wrong": 71, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 64, "jev_correct": 75, "local_accuracy": 0.64, "jev_accuracy": 0.75, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -20.0, -2.0 ], "local_only_correct": 5, "jev_only_correct": 16, "both_correct": 59, "both_wrong": 20, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 49, "jev_correct": 56, "local_accuracy": 0.49, "jev_accuracy": 0.56, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -16.0, 2.0 ], "local_only_correct": 8, "jev_only_correct": 15, "both_correct": 41, "both_wrong": 36, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 55, "jev_correct": 49, "local_accuracy": 0.55, "jev_accuracy": 0.49, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ -1.0, 13.0 ], "local_only_correct": 10, "jev_only_correct": 4, "both_correct": 45, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 47, "jev_correct": 48, "local_accuracy": 0.47, "jev_accuracy": 0.48, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -9.0, 7.0 ], "local_only_correct": 7, "jev_only_correct": 8, "both_correct": 40, "both_wrong": 45, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 76, "jev_correct": 79, "local_accuracy": 0.76, "jev_accuracy": 0.79, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -9.0, 3.0 ], "local_only_correct": 4, "jev_only_correct": 7, "both_correct": 72, "both_wrong": 17, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 4, "jev_correct": 6, "local_accuracy": 0.04, "jev_accuracy": 0.06, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -7.0, 3.0 ], "local_only_correct": 3, "jev_only_correct": 5, "both_correct": 1, "both_wrong": 91, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 58, "jev_correct": 64, "local_accuracy": 0.58, "jev_accuracy": 0.64, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -14.024999999999999, 2.0 ], "local_only_correct": 7, "jev_only_correct": 13, "both_correct": 51, "both_wrong": 29, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 39, "jev_correct": 37, "local_accuracy": 0.39, "jev_accuracy": 0.37, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -7.0, 12.0 ], "local_only_correct": 13, "jev_only_correct": 11, "both_correct": 26, "both_wrong": 50, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 8, "jev_correct": 16, "local_accuracy": 0.08, "jev_accuracy": 0.16, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -14.0, -2.0 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 7, "both_wrong": 83, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 7, "jev_correct": 34, "local_accuracy": 0.07, "jev_accuracy": 0.34, "difference_pp": -27.0, "paired_cluster_bootstrap95_pp": [ -37.0, -17.0 ], "local_only_correct": 3, "jev_only_correct": 30, "both_correct": 4, "both_wrong": 63, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 77, "jev_correct": 83, "local_accuracy": 0.77, "jev_accuracy": 0.83, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -12.0, -1.0 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 76, "both_wrong": 16, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 16, "jev_correct": 21, "local_accuracy": 0.16, "jev_accuracy": 0.21, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -14.0, 4.0 ], "local_only_correct": 8, "jev_only_correct": 13, "both_correct": 8, "both_wrong": 71, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 40, "jev_correct": 51, "local_accuracy": 0.4, "jev_accuracy": 0.51, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -22.0, 0.0 ], "local_only_correct": 11, "jev_only_correct": 22, "both_correct": 29, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 54, "jev_correct": 65, "local_accuracy": 0.54, "jev_accuracy": 0.65, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -18.0, -4.0 ], "local_only_correct": 2, "jev_only_correct": 13, "both_correct": 52, "both_wrong": 33, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 34, "jev_correct": 39, "local_accuracy": 0.34, "jev_accuracy": 0.39, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.0, 2.0 ], "local_only_correct": 4, "jev_only_correct": 9, "both_correct": 30, "both_wrong": 57, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 9, "jev_correct": 9, "local_accuracy": 0.09, "jev_accuracy": 0.09, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -6.0, 6.0 ], "local_only_correct": 5, "jev_only_correct": 5, "both_correct": 4, "both_wrong": 86, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 51, "jev_correct": 59, "local_accuracy": 0.51, "jev_accuracy": 0.59, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -18.0, 1.0 ], "local_only_correct": 7, "jev_only_correct": 15, "both_correct": 44, "both_wrong": 34, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 94, "jev_correct": 91, "local_accuracy": 0.94, "jev_accuracy": 0.91, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -2.0, 8.0 ], "local_only_correct": 5, "jev_only_correct": 2, "both_correct": 89, "both_wrong": 4, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 82, "jev_correct": 88, "local_accuracy": 0.82, "jev_accuracy": 0.88, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -13.0, 1.0 ], "local_only_correct": 3, "jev_only_correct": 9, "both_correct": 79, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 69, "jev_correct": 48, "local_accuracy": 0.69, "jev_accuracy": 0.48, "difference_pp": 21.0, "paired_cluster_bootstrap95_pp": [ 10.0, 33.0 ], "local_only_correct": 29, "jev_only_correct": 8, "both_correct": 40, "both_wrong": 23, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2423, "jev_correct": 2550, "local_accuracy": 0.8219131614654003, "jev_accuracy": 0.8649932157394844, "difference_pp": -4.308005427408412, "paired_cluster_bootstrap95_pp": [ -5.4613297150610585, -3.1207598371777476 ], "local_only_correct": 92, "jev_only_correct": 219, "both_correct": 2331, "both_wrong": 306, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2021, "jev_correct": 2412, "local_accuracy": 0.6855495251017639, "jev_accuracy": 0.8181818181818182, "difference_pp": -13.263229308005428, "paired_cluster_bootstrap95_pp": [ -14.824457259158752, -11.736770691994572 ], "local_only_correct": 94, "jev_only_correct": 485, "both_correct": 1927, "both_wrong": 442, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 793, "jev_correct": 843, "local_accuracy": 0.6760443307757886, "jev_accuracy": 0.7186700767263428, "difference_pp": -4.262574595055414, "paired_cluster_bootstrap95_pp": [ -8.981142666928932, 0.20142003103459835 ], "local_only_correct": 163, "jev_only_correct": 213, "both_correct": 630, "both_wrong": 167, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "decider-2b": { "fresh/news": { "n": 200, "local_correct": 176, "jev_correct": 178, "local_accuracy": 0.88, "jev_accuracy": 0.89, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 3.5 ], "local_only_correct": 8, "jev_only_correct": 10, "both_correct": 168, "both_wrong": 14, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 166, "jev_correct": 120, "local_accuracy": 0.83, "jev_accuracy": 0.6, "difference_pp": 23.0, "paired_cluster_bootstrap95_pp": [ 16.5, 30.5 ], "local_only_correct": 54, "jev_only_correct": 8, "both_correct": 112, "both_wrong": 26, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 196, "jev_correct": 196, "local_accuracy": 0.98, "jev_accuracy": 0.98, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -1.5, 1.5 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 195, "both_wrong": 3, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 189, "jev_correct": 179, "local_accuracy": 0.945, "jev_accuracy": 0.895, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ 0.0, 10.0 ], "local_only_correct": 17, "jev_only_correct": 7, "both_correct": 172, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 9, "jev_correct": 11, "local_accuracy": 0.75, "jev_accuracy": 0.9166666666666666, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 9, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 6, "jev_correct": 10, "local_accuracy": 0.5, "jev_accuracy": 0.8333333333333334, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -58.333333333333336, -8.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 6, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 11, "jev_correct": 12, "local_accuracy": 0.9166666666666666, "jev_accuracy": 1.0, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 11, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 93, "jev_correct": 91, "local_accuracy": 0.93, "jev_accuracy": 0.91, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -3.0, 7.0 ], "local_only_correct": 4, "jev_only_correct": 2, "both_correct": 89, "both_wrong": 5, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 76, "jev_correct": 48, "local_accuracy": 0.76, "jev_accuracy": 0.48, "difference_pp": 28.000000000000004, "paired_cluster_bootstrap95_pp": [ 18.0, 38.0 ], "local_only_correct": 32, "jev_only_correct": 4, "both_correct": 44, "both_wrong": 20, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 78, "jev_correct": 89, "local_accuracy": 0.78, "jev_accuracy": 0.89, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -19.0, -4.0 ], "local_only_correct": 2, "jev_only_correct": 13, "both_correct": 76, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 63, "jev_correct": 81, "local_accuracy": 0.5675675675675675, "jev_accuracy": 0.7297297297297297, "difference_pp": -16.216216216216218, "paired_cluster_bootstrap95_pp": [ -26.126126126126128, -6.306306306306307 ], "local_only_correct": 8, "jev_only_correct": 26, "both_correct": 55, "both_wrong": 22, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 64, "jev_correct": 70, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9722222222222222, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -18.055555555555557, 0.03472222222203274 ], "local_only_correct": 2, "jev_only_correct": 8, "both_correct": 62, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.7894736842105263, "jev_accuracy": 0.9473684210526315, "difference_pp": -15.789473684210526, "paired_cluster_bootstrap95_pp": [ -36.8421052631579, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.8125, "jev_accuracy": 1.0, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 17, "jev_correct": 20, "local_accuracy": 0.7727272727272727, "jev_accuracy": 0.9090909090909091, "difference_pp": -13.636363636363635, "paired_cluster_bootstrap95_pp": [ -31.818181818181817, 4.545454545454546 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 16, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.7222222222222222, "jev_accuracy": 1.0, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.7647058823529411, "jev_accuracy": 0.9411764705882353, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 11, "jev_correct": 18, "local_accuracy": 0.6111111111111112, "jev_accuracy": 1.0, "difference_pp": -38.88888888888889, "paired_cluster_bootstrap95_pp": [ -61.111111111111114, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 11, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 20, "jev_correct": 21, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9545454545454546, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 20, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7222222222222222, "jev_accuracy": 0.9444444444444444, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, 0.0 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7777777777777778, "jev_accuracy": 0.9444444444444444, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 19, "jev_correct": 18, "local_accuracy": 0.95, "jev_accuracy": 0.9, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ 0.0, 15.0 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 8, "jev_correct": 18, "local_accuracy": 0.38095238095238093, "jev_accuracy": 0.8571428571428571, "difference_pp": -47.61904761904761, "paired_cluster_bootstrap95_pp": [ -71.42857142857143, -23.69047619047684 ], "local_only_correct": 1, "jev_only_correct": 11, "both_correct": 7, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.8571428571428571, "jev_accuracy": 1.0, "difference_pp": -14.285714285714285, "paired_cluster_bootstrap95_pp": [ -35.714285714285715, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.75, "jev_accuracy": 0.875, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.6875, "jev_accuracy": 0.875, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7777777777777778, "jev_accuracy": 0.9444444444444444, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -38.888888888888886, 5.555555555555555 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.75, "jev_accuracy": 0.9375, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5333333333333333, "jev_accuracy": 1.0, "difference_pp": -46.666666666666664, "paired_cluster_bootstrap95_pp": [ -73.33333333333333, -20.0 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.6842105263157895, "jev_accuracy": 0.8947368421052632, "difference_pp": -21.052631578947366, "paired_cluster_bootstrap95_pp": [ -47.36842105263158, 0.0 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 16, "jev_correct": 19, "local_accuracy": 0.8, "jev_accuracy": 0.95, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -35.0, 5.0 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 15, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.45, "jev_accuracy": 0.8, "difference_pp": -35.0, "paired_cluster_bootstrap95_pp": [ -55.0, -15.0 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 4, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 13.636363636363637 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 434, "jev_correct": 629, "local_accuracy": 0.4822222222222222, "jev_accuracy": 0.6988888888888889, "difference_pp": -21.666666666666668, "paired_cluster_bootstrap95_pp": [ -26.78432343360993, -17.201716855614336 ], "local_only_correct": 49, "jev_only_correct": 244, "both_correct": 385, "both_wrong": 222, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 373, "jev_correct": 465, "local_accuracy": 0.6498257839721254, "jev_accuracy": 0.8101045296167247, "difference_pp": -16.02787456445993, "paired_cluster_bootstrap95_pp": [ -19.860627177700348, -12.195121951219512 ], "local_only_correct": 25, "jev_only_correct": 117, "both_correct": 348, "both_wrong": 84, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 269, "jev_correct": 490, "local_accuracy": 0.3842857142857143, "jev_accuracy": 0.7, "difference_pp": -31.571428571428573, "paired_cluster_bootstrap95_pp": [ -35.67541671368465, -27.50298830047387 ], "local_only_correct": 41, "jev_only_correct": 262, "both_correct": 228, "both_wrong": 169, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 46, "jev_correct": 97, "local_accuracy": 0.115, "jev_accuracy": 0.2425, "difference_pp": -12.75, "paired_cluster_bootstrap95_pp": [ -17.5, -8.0 ], "local_only_correct": 28, "jev_only_correct": 79, "both_correct": 18, "both_wrong": 275, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 36, "jev_correct": 40, "local_accuracy": 0.9, "jev_accuracy": 1.0, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -20.0, -2.5 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 36, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 35, "jev_correct": 40, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -22.562499999999996, -2.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 35, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 30, "jev_correct": 39, "local_accuracy": 0.75, "jev_accuracy": 0.975, "difference_pp": -22.5, "paired_cluster_bootstrap95_pp": [ -35.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 30, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 30, "jev_correct": 39, "local_accuracy": 0.75, "jev_accuracy": 0.975, "difference_pp": -22.5, "paired_cluster_bootstrap95_pp": [ -37.5, -10.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 30, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 340, "jev_correct": 237, "local_accuracy": 0.85, "jev_accuracy": 0.5925, "difference_pp": 25.75, "paired_cluster_bootstrap95_pp": [ 20.75, 30.75 ], "local_only_correct": 114, "jev_only_correct": 11, "both_correct": 226, "both_wrong": 49, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 268, "jev_correct": 277, "local_accuracy": 0.8933333333333333, "jev_accuracy": 0.9233333333333333, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -6.0, 0.0 ], "local_only_correct": 6, "jev_only_correct": 15, "both_correct": 262, "both_wrong": 17, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 358, "jev_correct": 377, "local_accuracy": 0.895, "jev_accuracy": 0.9425, "difference_pp": -4.75, "paired_cluster_bootstrap95_pp": [ -7.75, -2.0 ], "local_only_correct": 8, "jev_only_correct": 27, "both_correct": 350, "both_wrong": 15, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 356, "jev_correct": 346, "local_accuracy": 0.89, "jev_accuracy": 0.865, "difference_pp": 2.5, "paired_cluster_bootstrap95_pp": [ 0.0, 5.0 ], "local_only_correct": 20, "jev_only_correct": 10, "both_correct": 336, "both_wrong": 34, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 376, "jev_correct": 360, "local_accuracy": 0.94, "jev_accuracy": 0.9, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ 1.25, 7.0 ], "local_only_correct": 25, "jev_only_correct": 9, "both_correct": 351, "both_wrong": 15, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 173, "jev_correct": 225, "local_accuracy": 0.5766666666666667, "jev_accuracy": 0.75, "difference_pp": -17.333333333333336, "paired_cluster_bootstrap95_pp": [ -22.008333333333333, -12.0 ], "local_only_correct": 10, "jev_only_correct": 62, "both_correct": 163, "both_wrong": 65, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 391, "jev_correct": 393, "local_accuracy": 0.9775, "jev_accuracy": 0.9825, "difference_pp": -0.5, "paired_cluster_bootstrap95_pp": [ -2.0, 0.75 ], "local_only_correct": 3, "jev_only_correct": 5, "both_correct": 388, "both_wrong": 4, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 266, "jev_correct": 265, "local_accuracy": 0.665, "jev_accuracy": 0.6625, "difference_pp": 0.25, "paired_cluster_bootstrap95_pp": [ -3.75, 4.0 ], "local_only_correct": 32, "jev_only_correct": 31, "both_correct": 234, "both_wrong": 103, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 81, "jev_correct": 75, "local_accuracy": 0.81, "jev_accuracy": 0.75, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ -2.0, 14.0 ], "local_only_correct": 11, "jev_only_correct": 5, "both_correct": 70, "both_wrong": 14, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 55, "jev_correct": 56, "local_accuracy": 0.55, "jev_accuracy": 0.56, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -8.0, 6.0 ], "local_only_correct": 7, "jev_only_correct": 8, "both_correct": 48, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 60, "jev_correct": 49, "local_accuracy": 0.6, "jev_accuracy": 0.49, "difference_pp": 11.0, "paired_cluster_bootstrap95_pp": [ -1.0, 23.0 ], "local_only_correct": 27, "jev_only_correct": 16, "both_correct": 33, "both_wrong": 24, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 45, "jev_correct": 48, "local_accuracy": 0.45, "jev_accuracy": 0.48, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -9.0, 3.0 ], "local_only_correct": 3, "jev_only_correct": 6, "both_correct": 42, "both_wrong": 49, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 82, "jev_correct": 79, "local_accuracy": 0.82, "jev_accuracy": 0.79, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -1.0, 8.0 ], "local_only_correct": 4, "jev_only_correct": 1, "both_correct": 78, "both_wrong": 17, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 10, "jev_correct": 6, "local_accuracy": 0.1, "jev_accuracy": 0.06, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -1.0, 10.0 ], "local_only_correct": 6, "jev_only_correct": 2, "both_correct": 4, "both_wrong": 88, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 62, "jev_correct": 64, "local_accuracy": 0.62, "jev_accuracy": 0.64, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -10.0, 6.0 ], "local_only_correct": 8, "jev_only_correct": 10, "both_correct": 54, "both_wrong": 28, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 39, "jev_correct": 37, "local_accuracy": 0.39, "jev_accuracy": 0.37, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -7.0, 11.0 ], "local_only_correct": 12, "jev_only_correct": 10, "both_correct": 27, "both_wrong": 51, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 6, "jev_correct": 16, "local_accuracy": 0.06, "jev_accuracy": 0.16, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -17.0, -3.0 ], "local_only_correct": 2, "jev_only_correct": 12, "both_correct": 4, "both_wrong": 82, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 11, "jev_correct": 34, "local_accuracy": 0.11, "jev_accuracy": 0.34, "difference_pp": -23.0, "paired_cluster_bootstrap95_pp": [ -33.0, -13.0 ], "local_only_correct": 5, "jev_only_correct": 28, "both_correct": 6, "both_wrong": 61, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 80, "jev_correct": 83, "local_accuracy": 0.8, "jev_accuracy": 0.83, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -8.0, 2.0 ], "local_only_correct": 2, "jev_only_correct": 5, "both_correct": 78, "both_wrong": 15, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 21, "jev_correct": 21, "local_accuracy": 0.21, "jev_accuracy": 0.21, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -9.0, 9.0 ], "local_only_correct": 10, "jev_only_correct": 10, "both_correct": 11, "both_wrong": 69, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 44, "jev_correct": 51, "local_accuracy": 0.44, "jev_accuracy": 0.51, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -16.0, 2.0 ], "local_only_correct": 7, "jev_only_correct": 14, "both_correct": 37, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 58, "jev_correct": 65, "local_accuracy": 0.58, "jev_accuracy": 0.65, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -15.0, 1.0 ], "local_only_correct": 5, "jev_only_correct": 12, "both_correct": 53, "both_wrong": 30, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 38, "jev_correct": 39, "local_accuracy": 0.38, "jev_accuracy": 0.39, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 5.0 ], "local_only_correct": 4, "jev_only_correct": 5, "both_correct": 34, "both_wrong": 57, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 7, "jev_correct": 9, "local_accuracy": 0.07, "jev_accuracy": 0.09, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -8.0, 4.0 ], "local_only_correct": 4, "jev_only_correct": 6, "both_correct": 3, "both_wrong": 87, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 60, "jev_correct": 59, "local_accuracy": 0.6, "jev_accuracy": 0.59, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 10.0 ], "local_only_correct": 10, "jev_only_correct": 9, "both_correct": 50, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 94, "jev_correct": 91, "local_accuracy": 0.94, "jev_accuracy": 0.91, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -2.0, 8.0 ], "local_only_correct": 5, "jev_only_correct": 2, "both_correct": 89, "both_wrong": 4, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 85, "jev_correct": 88, "local_accuracy": 0.85, "jev_accuracy": 0.88, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -10.0, 3.0 ], "local_only_correct": 4, "jev_only_correct": 7, "both_correct": 81, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 79, "jev_correct": 48, "local_accuracy": 0.79, "jev_accuracy": 0.48, "difference_pp": 31.0, "paired_cluster_bootstrap95_pp": [ 20.0, 42.0 ], "local_only_correct": 36, "jev_only_correct": 5, "both_correct": 43, "both_wrong": 16, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2433, "jev_correct": 2550, "local_accuracy": 0.8253052917232022, "jev_accuracy": 0.8649932157394844, "difference_pp": -3.9687924016282223, "paired_cluster_bootstrap95_pp": [ -5.122116689280868, -2.8833107191316145 ], "local_only_correct": 72, "jev_only_correct": 189, "both_correct": 2361, "both_wrong": 326, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2099, "jev_correct": 2412, "local_accuracy": 0.7120081411126187, "jev_accuracy": 0.8181818181818182, "difference_pp": -10.617367706919946, "paired_cluster_bootstrap95_pp": [ -12.143826322930801, -9.12483039348711 ], "local_only_correct": 95, "jev_only_correct": 408, "both_correct": 2004, "both_wrong": 441, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 818, "jev_correct": 843, "local_accuracy": 0.6973572037510657, "jev_accuracy": 0.7186700767263428, "difference_pp": -2.131287297527707, "paired_cluster_bootstrap95_pp": [ -5.6338512172692505, 1.3400324371984274 ], "local_only_correct": 132, "jev_only_correct": 157, "both_correct": 686, "both_wrong": 198, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "decider-4b": { "fresh/news": { "n": 200, "local_correct": 180, "jev_correct": 178, "local_accuracy": 0.9, "jev_accuracy": 0.89, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -2.5, 5.0 ], "local_only_correct": 9, "jev_only_correct": 7, "both_correct": 171, "both_wrong": 13, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 153, "jev_correct": 120, "local_accuracy": 0.765, "jev_accuracy": 0.6, "difference_pp": 16.5, "paired_cluster_bootstrap95_pp": [ 11.0, 22.5 ], "local_only_correct": 37, "jev_only_correct": 4, "both_correct": 116, "both_wrong": 43, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 195, "jev_correct": 196, "local_accuracy": 0.975, "jev_accuracy": 0.98, "difference_pp": -0.5, "paired_cluster_bootstrap95_pp": [ -2.5, 1.0 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 194, "both_wrong": 3, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 194, "jev_correct": 179, "local_accuracy": 0.97, "jev_accuracy": 0.895, "difference_pp": 7.5, "paired_cluster_bootstrap95_pp": [ 3.0, 12.5 ], "local_only_correct": 19, "jev_only_correct": 4, "both_correct": 175, "both_wrong": 2, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 10, "jev_correct": 11, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9166666666666666, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 10, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 8, "jev_correct": 10, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.8333333333333334, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 8, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 11, "jev_correct": 12, "local_accuracy": 0.9166666666666666, "jev_accuracy": 1.0, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 11, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 96, "jev_correct": 91, "local_accuracy": 0.96, "jev_accuracy": 0.91, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -1.0, 11.0 ], "local_only_correct": 7, "jev_only_correct": 2, "both_correct": 89, "both_wrong": 2, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 71, "jev_correct": 48, "local_accuracy": 0.71, "jev_accuracy": 0.48, "difference_pp": 23.0, "paired_cluster_bootstrap95_pp": [ 13.0, 34.0 ], "local_only_correct": 28, "jev_only_correct": 5, "both_correct": 43, "both_wrong": 24, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 89, "jev_correct": 89, "local_accuracy": 0.89, "jev_accuracy": 0.89, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -6.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 4, "both_correct": 85, "both_wrong": 7, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 72, "jev_correct": 81, "local_accuracy": 0.6486486486486487, "jev_accuracy": 0.7297297297297297, "difference_pp": -8.108108108108109, "paired_cluster_bootstrap95_pp": [ -16.216216216216218, 0.0 ], "local_only_correct": 6, "jev_only_correct": 15, "both_correct": 66, "both_wrong": 24, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 71, "jev_correct": 70, "local_accuracy": 0.9861111111111112, "jev_accuracy": 0.9722222222222222, "difference_pp": 1.3888888888888888, "paired_cluster_bootstrap95_pp": [ -2.7777777777777777, 5.555555555555555 ], "local_only_correct": 2, "jev_only_correct": 1, "both_correct": 69, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.9473684210526315, "jev_accuracy": 0.9473684210526315, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9090909090909091, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -22.727272727272727, 9.090909090909092 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 18, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.8235294117647058, "jev_accuracy": 0.9411764705882353, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 11.764705882352942 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 21, "jev_correct": 21, "local_accuracy": 0.9545454545454546, "jev_accuracy": 0.9545454545454546, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 13.636363636363637 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 20, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9444444444444444, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -22.22222222222222, 11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9444444444444444, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.9, "jev_accuracy": 0.9, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.6190476190476191, "jev_accuracy": 0.8571428571428571, "difference_pp": -23.809523809523807, "paired_cluster_bootstrap95_pp": [ -47.61904761904762, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 12, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 16, "jev_correct": 14, "local_accuracy": 1.0, "jev_accuracy": 0.875, "difference_pp": 12.5, "paired_cluster_bootstrap95_pp": [ 0.0, 31.25 ], "local_only_correct": 2, "jev_only_correct": 0, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.75, "jev_accuracy": 0.875, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/news_topic": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9444444444444444, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8125, "jev_accuracy": 0.9375, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -37.5, 12.5 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7368421052631579, "jev_accuracy": 0.8947368421052632, "difference_pp": -15.789473684210526, "paired_cluster_bootstrap95_pp": [ -31.57894736842105, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 2, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 17, "jev_correct": 19, "local_accuracy": 0.85, "jev_accuracy": 0.95, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -30.0, 10.0 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 16, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 0.8, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 4, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 531, "jev_correct": 629, "local_accuracy": 0.59, "jev_accuracy": 0.6988888888888889, "difference_pp": -10.888888888888888, "paired_cluster_bootstrap95_pp": [ -14.597733577843607, -7.359428700323997 ], "local_only_correct": 58, "jev_only_correct": 156, "both_correct": 473, "both_wrong": 213, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 415, "jev_correct": 465, "local_accuracy": 0.7229965156794426, "jev_accuracy": 0.8101045296167247, "difference_pp": -8.710801393728223, "paired_cluster_bootstrap95_pp": [ -12.369337979094077, -5.2264808362369335 ], "local_only_correct": 29, "jev_only_correct": 79, "both_correct": 386, "both_wrong": 80, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 363, "jev_correct": 490, "local_accuracy": 0.5185714285714286, "jev_accuracy": 0.7, "difference_pp": -18.142857142857142, "paired_cluster_bootstrap95_pp": [ -22.351545231273025, -13.976945244956772 ], "local_only_correct": 59, "jev_only_correct": 186, "both_correct": 304, "both_wrong": 151, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 66, "jev_correct": 97, "local_accuracy": 0.165, "jev_accuracy": 0.2425, "difference_pp": -7.75, "paired_cluster_bootstrap95_pp": [ -12.75, -2.75 ], "local_only_correct": 40, "jev_only_correct": 71, "both_correct": 26, "both_wrong": 263, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en/reversed": { "n": 40, "local_correct": 38, "jev_correct": 40, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 38, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 38, "jev_correct": 40, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 38, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 31, "jev_correct": 39, "local_accuracy": 0.775, "jev_accuracy": 0.975, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -32.5, -7.5 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 31, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 33, "jev_correct": 39, "local_accuracy": 0.825, "jev_accuracy": 0.975, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -27.5, -5.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 33, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 326, "jev_correct": 237, "local_accuracy": 0.815, "jev_accuracy": 0.5925, "difference_pp": 22.25, "paired_cluster_bootstrap95_pp": [ 17.5, 27.006249999999966 ], "local_only_correct": 99, "jev_only_correct": 10, "both_correct": 227, "both_wrong": 64, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 280, "jev_correct": 277, "local_accuracy": 0.9333333333333333, "jev_accuracy": 0.9233333333333333, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -1.6666666666666667, 3.6666666666666665 ], "local_only_correct": 10, "jev_only_correct": 7, "both_correct": 270, "both_wrong": 13, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 373, "jev_correct": 377, "local_accuracy": 0.9325, "jev_accuracy": 0.9425, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -3.0, 0.75 ], "local_only_correct": 5, "jev_only_correct": 9, "both_correct": 368, "both_wrong": 18, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 365, "jev_correct": 346, "local_accuracy": 0.9125, "jev_accuracy": 0.865, "difference_pp": 4.75, "paired_cluster_bootstrap95_pp": [ 2.0, 7.5 ], "local_only_correct": 25, "jev_only_correct": 6, "both_correct": 340, "both_wrong": 29, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 384, "jev_correct": 360, "local_accuracy": 0.96, "jev_accuracy": 0.9, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ 2.7437500000000004, 9.5 ], "local_only_correct": 37, "jev_only_correct": 13, "both_correct": 347, "both_wrong": 3, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 224, "jev_correct": 225, "local_accuracy": 0.7466666666666667, "jev_accuracy": 0.75, "difference_pp": -0.33333333333333337, "paired_cluster_bootstrap95_pp": [ -3.3333333333333335, 2.6666666666666665 ], "local_only_correct": 11, "jev_only_correct": 12, "both_correct": 213, "both_wrong": 64, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 395, "jev_correct": 393, "local_accuracy": 0.9875, "jev_accuracy": 0.9825, "difference_pp": 0.5, "paired_cluster_bootstrap95_pp": [ -0.75, 1.75 ], "local_only_correct": 4, "jev_only_correct": 2, "both_correct": 391, "both_wrong": 3, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 311, "jev_correct": 265, "local_accuracy": 0.7775, "jev_accuracy": 0.6625, "difference_pp": 11.5, "paired_cluster_bootstrap95_pp": [ 8.0, 15.25 ], "local_only_correct": 58, "jev_only_correct": 12, "both_correct": 253, "both_wrong": 77, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 77, "jev_correct": 75, "local_accuracy": 0.77, "jev_accuracy": 0.75, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -5.0, 9.0 ], "local_only_correct": 7, "jev_only_correct": 5, "both_correct": 70, "both_wrong": 18, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 55, "jev_correct": 56, "local_accuracy": 0.55, "jev_accuracy": 0.56, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -8.0, 6.0 ], "local_only_correct": 6, "jev_only_correct": 7, "both_correct": 49, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 48, "jev_correct": 49, "local_accuracy": 0.48, "jev_accuracy": 0.49, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -10.0, 8.0 ], "local_only_correct": 9, "jev_only_correct": 10, "both_correct": 39, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 43, "jev_correct": 48, "local_accuracy": 0.43, "jev_accuracy": 0.48, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.0, 1.0 ], "local_only_correct": 3, "jev_only_correct": 8, "both_correct": 40, "both_wrong": 49, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 80, "jev_correct": 79, "local_accuracy": 0.8, "jev_accuracy": 0.79, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -3.0, 6.0 ], "local_only_correct": 3, "jev_only_correct": 2, "both_correct": 77, "both_wrong": 18, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 6, "jev_correct": 6, "local_accuracy": 0.06, "jev_accuracy": 0.06, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -5.0, 5.0 ], "local_only_correct": 4, "jev_only_correct": 4, "both_correct": 2, "both_wrong": 90, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 67, "jev_correct": 64, "local_accuracy": 0.67, "jev_accuracy": 0.64, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -5.0, 12.0 ], "local_only_correct": 11, "jev_only_correct": 8, "both_correct": 56, "both_wrong": 25, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 41, "jev_correct": 37, "local_accuracy": 0.41, "jev_accuracy": 0.37, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -4.0, 12.0 ], "local_only_correct": 10, "jev_only_correct": 6, "both_correct": 31, "both_wrong": 53, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.15, "jev_accuracy": 0.16, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 4.0 ], "local_only_correct": 4, "jev_only_correct": 5, "both_correct": 11, "both_wrong": 80, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 26, "jev_correct": 34, "local_accuracy": 0.26, "jev_accuracy": 0.34, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -17.0, 0.0 ], "local_only_correct": 5, "jev_only_correct": 13, "both_correct": 21, "both_wrong": 61, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 84, "jev_correct": 83, "local_accuracy": 0.84, "jev_accuracy": 0.83, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -3.0, 6.0 ], "local_only_correct": 3, "jev_only_correct": 2, "both_correct": 81, "both_wrong": 14, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 24, "jev_correct": 21, "local_accuracy": 0.24, "jev_accuracy": 0.21, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -5.0, 11.0 ], "local_only_correct": 10, "jev_only_correct": 7, "both_correct": 14, "both_wrong": 69, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 44, "jev_correct": 51, "local_accuracy": 0.44, "jev_accuracy": 0.51, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -15.0, 2.0 ], "local_only_correct": 6, "jev_only_correct": 13, "both_correct": 38, "both_wrong": 43, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 65, "jev_correct": 65, "local_accuracy": 0.65, "jev_accuracy": 0.65, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -6.0, 5.0 ], "local_only_correct": 4, "jev_only_correct": 4, "both_correct": 61, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 33, "jev_correct": 39, "local_accuracy": 0.33, "jev_accuracy": 0.39, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -13.0, 0.0 ], "local_only_correct": 3, "jev_only_correct": 9, "both_correct": 30, "both_wrong": 58, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 6, "jev_correct": 9, "local_accuracy": 0.06, "jev_accuracy": 0.09, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -8.0, 2.0 ], "local_only_correct": 2, "jev_only_correct": 5, "both_correct": 4, "both_wrong": 89, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 66, "jev_correct": 59, "local_accuracy": 0.66, "jev_accuracy": 0.59, "difference_pp": 7.000000000000001, "paired_cluster_bootstrap95_pp": [ 1.0, 14.0 ], "local_only_correct": 9, "jev_only_correct": 2, "both_correct": 57, "both_wrong": 32, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 96, "jev_correct": 91, "local_accuracy": 0.96, "jev_accuracy": 0.91, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -2.0, 11.0 ], "local_only_correct": 8, "jev_only_correct": 3, "both_correct": 88, "both_wrong": 1, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 87, "jev_correct": 88, "local_accuracy": 0.87, "jev_accuracy": 0.88, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 3.0 ], "local_only_correct": 2, "jev_only_correct": 3, "both_correct": 85, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 77, "jev_correct": 48, "local_accuracy": 0.77, "jev_accuracy": 0.48, "difference_pp": 28.999999999999996, "paired_cluster_bootstrap95_pp": [ 18.0, 40.0 ], "local_only_correct": 34, "jev_only_correct": 5, "both_correct": 43, "both_wrong": 18, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2603, "jev_correct": 2550, "local_accuracy": 0.8829715061058344, "jev_accuracy": 0.8649932157394844, "difference_pp": 1.797829036635007, "paired_cluster_bootstrap95_pp": [ 0.9497964721845319, 2.645861601085482 ], "local_only_correct": 112, "jev_only_correct": 59, "both_correct": 2491, "both_wrong": 286, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2418, "jev_correct": 2412, "local_accuracy": 0.8202170963364993, "jev_accuracy": 0.8181818181818182, "difference_pp": 0.20352781546811397, "paired_cluster_bootstrap95_pp": [ -0.9158751696065129, 1.2890094979647218 ], "local_only_correct": 145, "jev_only_correct": 139, "both_correct": 2273, "both_wrong": 391, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 850, "jev_correct": 843, "local_accuracy": 0.7246376811594203, "jev_accuracy": 0.7186700767263428, "difference_pp": 0.5967604433077579, "paired_cluster_bootstrap95_pp": [ -2.1825349204345836, 3.285077667430608 ], "local_only_correct": 135, "jev_only_correct": 128, "both_correct": 715, "both_wrong": 195, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "decision-4b-v12": { "fresh/news": { "n": 200, "local_correct": 174, "jev_correct": 178, "local_accuracy": 0.87, "jev_accuracy": 0.89, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -4.5, 0.0 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 173, "both_wrong": 21, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 116, "jev_correct": 120, "local_accuracy": 0.58, "jev_accuracy": 0.6, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -7.0, 3.5 ], "local_only_correct": 12, "jev_only_correct": 16, "both_correct": 104, "both_wrong": 68, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 189, "jev_correct": 196, "local_accuracy": 0.945, "jev_accuracy": 0.98, "difference_pp": -3.5000000000000004, "paired_cluster_bootstrap95_pp": [ -6.5, -1.0 ], "local_only_correct": 1, "jev_only_correct": 8, "both_correct": 188, "both_wrong": 3, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 185, "jev_correct": 179, "local_accuracy": 0.925, "jev_accuracy": 0.895, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -2.0, 8.0 ], "local_only_correct": 15, "jev_only_correct": 9, "both_correct": 170, "both_wrong": 6, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 11, "jev_correct": 11, "local_accuracy": 0.9166666666666666, "jev_accuracy": 0.9166666666666666, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -25.0, 25.0 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 10, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 9, "jev_correct": 10, "local_accuracy": 0.75, "jev_accuracy": 0.8333333333333334, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 9, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 91, "jev_correct": 91, "local_accuracy": 0.91, "jev_accuracy": 0.91, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -3.0, 3.0 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 90, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 51, "jev_correct": 48, "local_accuracy": 0.51, "jev_accuracy": 0.48, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -5.0, 11.0 ], "local_only_correct": 10, "jev_only_correct": 7, "both_correct": 41, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 0, "jev_correct": 89, "local_accuracy": 0.0, "jev_accuracy": 0.89, "difference_pp": -89.0, "paired_cluster_bootstrap95_pp": [ -95.0, -82.0 ], "local_only_correct": 0, "jev_only_correct": 89, "both_correct": 0, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 86, "jev_correct": 81, "local_accuracy": 0.7747747747747747, "jev_accuracy": 0.7297297297297297, "difference_pp": 4.504504504504505, "paired_cluster_bootstrap95_pp": [ -3.6036036036036037, 12.612612612612613 ], "local_only_correct": 13, "jev_only_correct": 8, "both_correct": 73, "both_wrong": 17, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 69, "jev_correct": 70, "local_accuracy": 0.9583333333333334, "jev_accuracy": 0.9722222222222222, "difference_pp": -1.3888888888888888, "paired_cluster_bootstrap95_pp": [ -6.944444444444445, 2.7777777777777777 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 68, "both_wrong": 1, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.9473684210526315, "jev_accuracy": 0.9473684210526315, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 15.789473684210526 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 18, "jev_correct": 20, "local_accuracy": 0.8181818181818182, "jev_accuracy": 0.9090909090909091, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -22.727272727272727, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 18, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.8823529411764706, "jev_accuracy": 0.9411764705882353, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 19, "jev_correct": 21, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9545454545454546, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -22.727272727272727, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 17, "jev_correct": 17, "local_accuracy": 0.9444444444444444, "jev_accuracy": 0.9444444444444444, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 16.666666666666668 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.6875, "jev_accuracy": 0.9375, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 17, "jev_correct": 17, "local_accuracy": 0.9444444444444444, "jev_accuracy": 0.9444444444444444, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8, "jev_accuracy": 0.9, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 10, "jev_correct": 18, "local_accuracy": 0.47619047619047616, "jev_accuracy": 0.8571428571428571, "difference_pp": -38.095238095238095, "paired_cluster_bootstrap95_pp": [ -61.904761904761905, -14.285714285714286 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 9, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.8571428571428571, "jev_accuracy": 1.0, "difference_pp": -14.285714285714285, "paired_cluster_bootstrap95_pp": [ -35.714285714285715, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 14, "jev_correct": 14, "local_accuracy": 0.875, "jev_accuracy": 0.875, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -18.75, 18.75 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 13, "jev_correct": 14, "local_accuracy": 0.8125, "jev_accuracy": 0.875, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -25.0, 12.5 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6111111111111112, "jev_accuracy": 0.9444444444444444, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8125, "jev_accuracy": 0.9375, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -37.5, 12.5 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.631578947368421, "jev_accuracy": 0.8947368421052632, "difference_pp": -26.31578947368421, "paired_cluster_bootstrap95_pp": [ -47.36842105263158, -5.2631578947368425 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 2, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 13, "jev_correct": 19, "local_accuracy": 0.65, "jev_accuracy": 0.95, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -50.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 13, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 0.8, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -25.0, 15.0 ], "local_only_correct": 2, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 13.636363636363637 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 578, "jev_correct": 629, "local_accuracy": 0.6422222222222222, "jev_accuracy": 0.6988888888888889, "difference_pp": -5.666666666666666, "paired_cluster_bootstrap95_pp": [ -9.629852744310575, -2.237908839986976 ], "local_only_correct": 82, "jev_only_correct": 133, "both_correct": 496, "both_wrong": 189, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 413, "jev_correct": 465, "local_accuracy": 0.7195121951219512, "jev_accuracy": 0.8101045296167247, "difference_pp": -9.059233449477352, "paired_cluster_bootstrap95_pp": [ -12.717770034843205, -5.400696864111498 ], "local_only_correct": 31, "jev_only_correct": 83, "both_correct": 382, "both_wrong": 78, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 364, "jev_correct": 490, "local_accuracy": 0.52, "jev_accuracy": 0.7, "difference_pp": -18.0, "paired_cluster_bootstrap95_pp": [ -22.22300469483568, -13.96933479921032 ], "local_only_correct": 52, "jev_only_correct": 178, "both_correct": 312, "both_wrong": 158, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 69, "jev_correct": 97, "local_accuracy": 0.1725, "jev_accuracy": 0.2425, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -11.75, -1.993750000000034 ], "local_only_correct": 38, "jev_only_correct": 66, "both_correct": 31, "both_wrong": 265, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb": { "n": 28, "local_correct": 24, "jev_correct": 28, "local_accuracy": 0.8571428571428571, "jev_accuracy": 1.0, "difference_pp": -14.285714285714285, "paired_cluster_bootstrap95_pp": [ -28.571428571428573, -3.5714285714285716 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 24, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en/reversed": { "n": 40, "local_correct": 40, "jev_correct": 40, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 40, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en": { "n": 40, "local_correct": 40, "jev_correct": 40, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 40, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 37, "jev_correct": 39, "local_accuracy": 0.925, "jev_accuracy": 0.975, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 37, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 38, "jev_correct": 39, "local_accuracy": 0.95, "jev_accuracy": 0.975, "difference_pp": -2.5, "paired_cluster_bootstrap95_pp": [ -7.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 38, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 217, "jev_correct": 237, "local_accuracy": 0.5425, "jev_accuracy": 0.5925, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -8.5, -1.25 ], "local_only_correct": 21, "jev_only_correct": 41, "both_correct": 196, "both_wrong": 142, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 274, "jev_correct": 277, "local_accuracy": 0.9133333333333333, "jev_accuracy": 0.9233333333333333, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -3.6666666666666665, 1.6666666666666667 ], "local_only_correct": 6, "jev_only_correct": 9, "both_correct": 268, "both_wrong": 17, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 335, "jev_correct": 377, "local_accuracy": 0.8375, "jev_accuracy": 0.9425, "difference_pp": -10.5, "paired_cluster_bootstrap95_pp": [ -14.0, -7.5 ], "local_only_correct": 3, "jev_only_correct": 45, "both_correct": 332, "both_wrong": 20, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 339, "jev_correct": 346, "local_accuracy": 0.8475, "jev_accuracy": 0.865, "difference_pp": -1.7500000000000002, "paired_cluster_bootstrap95_pp": [ -4.0, 0.5 ], "local_only_correct": 7, "jev_only_correct": 14, "both_correct": 332, "both_wrong": 47, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 366, "jev_correct": 360, "local_accuracy": 0.915, "jev_accuracy": 0.9, "difference_pp": 1.5, "paired_cluster_bootstrap95_pp": [ -1.5, 4.5 ], "local_only_correct": 21, "jev_only_correct": 15, "both_correct": 345, "both_wrong": 19, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 235, "jev_correct": 225, "local_accuracy": 0.7833333333333333, "jev_accuracy": 0.75, "difference_pp": 3.3333333333333335, "paired_cluster_bootstrap95_pp": [ 0.0, 6.666666666666667 ], "local_only_correct": 18, "jev_only_correct": 8, "both_correct": 217, "both_wrong": 57, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 381, "jev_correct": 393, "local_accuracy": 0.9525, "jev_accuracy": 0.9825, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -4.50625, -1.5 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 381, "both_wrong": 7, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 309, "jev_correct": 265, "local_accuracy": 0.7725, "jev_accuracy": 0.6625, "difference_pp": 11.0, "paired_cluster_bootstrap95_pp": [ 6.25, 15.5 ], "local_only_correct": 67, "jev_only_correct": 23, "both_correct": 242, "both_wrong": 68, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 70, "jev_correct": 75, "local_accuracy": 0.7, "jev_accuracy": 0.75, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -11.0, 1.0 ], "local_only_correct": 2, "jev_only_correct": 7, "both_correct": 68, "both_wrong": 23, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 48, "jev_correct": 56, "local_accuracy": 0.48, "jev_accuracy": 0.56, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -17.0, 1.0 ], "local_only_correct": 6, "jev_only_correct": 14, "both_correct": 42, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 48, "jev_correct": 49, "local_accuracy": 0.48, "jev_accuracy": 0.49, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 5.0 ], "local_only_correct": 4, "jev_only_correct": 5, "both_correct": 44, "both_wrong": 47, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 39, "jev_correct": 48, "local_accuracy": 0.39, "jev_accuracy": 0.48, "difference_pp": -9.0, "paired_cluster_bootstrap95_pp": [ -19.0, 1.0 ], "local_only_correct": 8, "jev_only_correct": 17, "both_correct": 31, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 82, "jev_correct": 79, "local_accuracy": 0.82, "jev_accuracy": 0.79, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -1.0, 8.0 ], "local_only_correct": 4, "jev_only_correct": 1, "both_correct": 78, "both_wrong": 17, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 10, "jev_correct": 6, "local_accuracy": 0.1, "jev_accuracy": 0.06, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -1.0, 10.0 ], "local_only_correct": 6, "jev_only_correct": 2, "both_correct": 4, "both_wrong": 88, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 64, "jev_correct": 64, "local_accuracy": 0.64, "jev_accuracy": 0.64, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -8.0, 9.0 ], "local_only_correct": 9, "jev_only_correct": 9, "both_correct": 55, "both_wrong": 27, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 41, "jev_correct": 37, "local_accuracy": 0.41, "jev_accuracy": 0.37, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -3.0, 12.0 ], "local_only_correct": 9, "jev_only_correct": 5, "both_correct": 32, "both_wrong": 54, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 5, "jev_correct": 16, "local_accuracy": 0.05, "jev_accuracy": 0.16, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -19.0, -3.0 ], "local_only_correct": 3, "jev_only_correct": 14, "both_correct": 2, "both_wrong": 81, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 30, "jev_correct": 34, "local_accuracy": 0.3, "jev_accuracy": 0.34, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -15.0, 7.0 ], "local_only_correct": 13, "jev_only_correct": 17, "both_correct": 17, "both_wrong": 53, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 88, "jev_correct": 83, "local_accuracy": 0.88, "jev_accuracy": 0.83, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -2.0, 13.0 ], "local_only_correct": 10, "jev_only_correct": 5, "both_correct": 78, "both_wrong": 7, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 8, "jev_correct": 21, "local_accuracy": 0.08, "jev_accuracy": 0.21, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -22.0, -4.0 ], "local_only_correct": 5, "jev_only_correct": 18, "both_correct": 3, "both_wrong": 74, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 46, "jev_correct": 51, "local_accuracy": 0.46, "jev_accuracy": 0.51, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -14.0, 4.0 ], "local_only_correct": 7, "jev_only_correct": 12, "both_correct": 39, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 0, "jev_correct": 65, "local_accuracy": 0.0, "jev_accuracy": 0.65, "difference_pp": -65.0, "paired_cluster_bootstrap95_pp": [ -75.0, -56.0 ], "local_only_correct": 0, "jev_only_correct": 65, "both_correct": 0, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 34, "jev_correct": 39, "local_accuracy": 0.34, "jev_accuracy": 0.39, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.0, 2.0 ], "local_only_correct": 4, "jev_only_correct": 9, "both_correct": 30, "both_wrong": 57, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 8, "jev_correct": 9, "local_accuracy": 0.08, "jev_accuracy": 0.09, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 5.0 ], "local_only_correct": 4, "jev_only_correct": 5, "both_correct": 4, "both_wrong": 87, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 58, "jev_correct": 59, "local_accuracy": 0.58, "jev_accuracy": 0.59, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 5.0 ], "local_only_correct": 4, "jev_only_correct": 5, "both_correct": 54, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 88, "jev_correct": 91, "local_accuracy": 0.88, "jev_accuracy": 0.91, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -7.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 88, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 0, "jev_correct": 88, "local_accuracy": 0.0, "jev_accuracy": 0.88, "difference_pp": -88.0, "paired_cluster_bootstrap95_pp": [ -94.0, -81.0 ], "local_only_correct": 0, "jev_only_correct": 88, "both_correct": 0, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 54, "jev_correct": 48, "local_accuracy": 0.54, "jev_accuracy": 0.48, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ -3.0, 15.0 ], "local_only_correct": 13, "jev_only_correct": 7, "both_correct": 41, "both_wrong": 39, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2425, "jev_correct": 2550, "local_accuracy": 0.8225915875169606, "jev_accuracy": 0.8649932157394844, "difference_pp": -4.240162822252374, "paired_cluster_bootstrap95_pp": [ -5.325644504748983, -3.1546811397557666 ], "local_only_correct": 77, "jev_only_correct": 202, "both_correct": 2348, "both_wrong": 321, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2150, "jev_correct": 2412, "local_accuracy": 0.7293080054274084, "jev_accuracy": 0.8181818181818182, "difference_pp": -8.887381275440976, "paired_cluster_bootstrap95_pp": [ -10.278154681139755, -7.530529172320217 ], "local_only_correct": 79, "jev_only_correct": 341, "both_correct": 2071, "both_wrong": 457, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 787, "jev_correct": 843, "local_accuracy": 0.670929241261722, "jev_accuracy": 0.7186700767263428, "difference_pp": -4.774083546462063, "paired_cluster_bootstrap95_pp": [ -7.248064301212738, -2.2763936159666938 ], "local_only_correct": 90, "jev_only_correct": 146, "both_correct": 697, "both_wrong": 240, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "decision2-eos-08b": { "fresh/news": { "n": 200, "local_correct": 165, "jev_correct": 178, "local_accuracy": 0.825, "jev_accuracy": 0.89, "difference_pp": -6.5, "paired_cluster_bootstrap95_pp": [ -11.0, -2.0 ], "local_only_correct": 4, "jev_only_correct": 17, "both_correct": 161, "both_wrong": 18, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 102, "jev_correct": 120, "local_accuracy": 0.51, "jev_accuracy": 0.6, "difference_pp": -9.0, "paired_cluster_bootstrap95_pp": [ -16.0, -1.5 ], "local_only_correct": 19, "jev_only_correct": 37, "both_correct": 83, "both_wrong": 61, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 139, "jev_correct": 196, "local_accuracy": 0.695, "jev_accuracy": 0.98, "difference_pp": -28.499999999999996, "paired_cluster_bootstrap95_pp": [ -35.0, -22.5 ], "local_only_correct": 1, "jev_only_correct": 58, "both_correct": 138, "both_wrong": 3, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 152, "jev_correct": 179, "local_accuracy": 0.76, "jev_accuracy": 0.895, "difference_pp": -13.5, "paired_cluster_bootstrap95_pp": [ -20.5, -6.987500000000068 ], "local_only_correct": 13, "jev_only_correct": 40, "both_correct": 139, "both_wrong": 8, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 6, "jev_correct": 11, "local_accuracy": 0.5, "jev_accuracy": 0.9166666666666666, "difference_pp": -41.66666666666667, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 6, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 5, "jev_correct": 10, "local_accuracy": 0.4166666666666667, "jev_accuracy": 0.8333333333333334, "difference_pp": -41.66666666666667, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 5, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 10, "jev_correct": 12, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 10, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 10, "jev_correct": 12, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 10, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 73, "jev_correct": 91, "local_accuracy": 0.73, "jev_accuracy": 0.91, "difference_pp": -18.0, "paired_cluster_bootstrap95_pp": [ -26.0, -11.0 ], "local_only_correct": 0, "jev_only_correct": 18, "both_correct": 73, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 49, "jev_correct": 48, "local_accuracy": 0.49, "jev_accuracy": 0.48, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -10.0, 12.0 ], "local_only_correct": 15, "jev_only_correct": 14, "both_correct": 34, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 90, "jev_correct": 89, "local_accuracy": 0.9, "jev_accuracy": 0.89, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -6.0, 9.0 ], "local_only_correct": 7, "jev_only_correct": 6, "both_correct": 83, "both_wrong": 4, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 51, "jev_correct": 81, "local_accuracy": 0.4594594594594595, "jev_accuracy": 0.7297297297297297, "difference_pp": -27.027027027027028, "paired_cluster_bootstrap95_pp": [ -37.83783783783784, -16.216216216216218 ], "local_only_correct": 7, "jev_only_correct": 37, "both_correct": 44, "both_wrong": 23, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 64, "jev_correct": 70, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9722222222222222, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, -1.3888888888888888 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 63, "both_wrong": 1, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8421052631578947, "jev_accuracy": 0.9473684210526315, "difference_pp": -10.526315789473683, "paired_cluster_bootstrap95_pp": [ -26.31578947368421, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 17, "jev_correct": 20, "local_accuracy": 0.7727272727272727, "jev_accuracy": 0.9090909090909091, "difference_pp": -13.636363636363635, "paired_cluster_bootstrap95_pp": [ -31.818181818181817, 4.545454545454546 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 16, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.8235294117647058, "jev_accuracy": 0.9411764705882353, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 11.764705882352942 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.7333333333333333, "jev_accuracy": 1.0, "difference_pp": -26.666666666666668, "paired_cluster_bootstrap95_pp": [ -53.333333333333336, -6.666666666666667 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.75, "jev_accuracy": 0.9375, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 11, "jev_correct": 18, "local_accuracy": 0.6111111111111112, "jev_accuracy": 1.0, "difference_pp": -38.88888888888889, "paired_cluster_bootstrap95_pp": [ -61.111111111111114, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 11, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 19, "jev_correct": 21, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9545454545454546, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -22.727272727272727, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 16, "jev_correct": 19, "local_accuracy": 0.8421052631578947, "jev_accuracy": 1.0, "difference_pp": -15.789473684210526, "paired_cluster_bootstrap95_pp": [ -31.57894736842105, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 16, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9444444444444444, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -22.22222222222222, 11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5, "jev_accuracy": 0.9375, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7222222222222222, "jev_accuracy": 0.9444444444444444, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7, "jev_accuracy": 0.9, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, -5.0 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 6, "jev_correct": 18, "local_accuracy": 0.2857142857142857, "jev_accuracy": 0.8571428571428571, "difference_pp": -57.14285714285714, "paired_cluster_bootstrap95_pp": [ -80.95238095238095, -33.333333333333336 ], "local_only_correct": 1, "jev_only_correct": 13, "both_correct": 5, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 22, "jev_correct": 23, "local_accuracy": 0.9565217391304348, "jev_accuracy": 1.0, "difference_pp": -4.3478260869565215, "paired_cluster_bootstrap95_pp": [ -13.043478260869565, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 22, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.8571428571428571, "jev_accuracy": 1.0, "difference_pp": -14.285714285714285, "paired_cluster_bootstrap95_pp": [ -35.714285714285715, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.75, "jev_accuracy": 0.875, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 10, "jev_correct": 14, "local_accuracy": 0.625, "jev_accuracy": 0.875, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, 0.15624999999914735 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 9, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -43.75, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -43.75, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 7, "jev_correct": 17, "local_accuracy": 0.3888888888888889, "jev_accuracy": 0.9444444444444444, "difference_pp": -55.55555555555556, "paired_cluster_bootstrap95_pp": [ -77.77777777777777, -33.333333333333336 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 7, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.6875, "jev_accuracy": 0.9375, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, 0.0 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4666666666666667, "jev_accuracy": 1.0, "difference_pp": -53.333333333333336, "paired_cluster_bootstrap95_pp": [ -80.0, -26.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.631578947368421, "jev_accuracy": 0.8947368421052632, "difference_pp": -26.31578947368421, "paired_cluster_bootstrap95_pp": [ -52.63157894736842, 5.2631578947368425 ], "local_only_correct": 2, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 13, "jev_correct": 19, "local_accuracy": 0.65, "jev_accuracy": 0.95, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -50.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 13, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 20, "jev_correct": 20, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.6, "jev_accuracy": 0.8, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -45.0, 0.0 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 11, "both_wrong": 3, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -18.181818181818183, 18.181818181818183 ], "local_only_correct": 2, "jev_only_correct": 2, "both_correct": 18, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 438, "jev_correct": 629, "local_accuracy": 0.4866666666666667, "jev_accuracy": 0.6988888888888889, "difference_pp": -21.22222222222222, "paired_cluster_bootstrap95_pp": [ -26.659923928077458, -16.231834571678327 ], "local_only_correct": 71, "jev_only_correct": 262, "both_correct": 367, "both_wrong": 200, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 291, "jev_correct": 465, "local_accuracy": 0.5069686411149826, "jev_accuracy": 0.8101045296167247, "difference_pp": -30.313588850174217, "paired_cluster_bootstrap95_pp": [ -34.66898954703833, -25.78397212543554 ], "local_only_correct": 26, "jev_only_correct": 200, "both_correct": 265, "both_wrong": 83, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 161, "jev_correct": 490, "local_accuracy": 0.23, "jev_accuracy": 0.7, "difference_pp": -47.0, "paired_cluster_bootstrap95_pp": [ -51.30067227946721, -42.79599028970553 ], "local_only_correct": 28, "jev_only_correct": 357, "both_correct": 133, "both_wrong": 182, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 8, "jev_correct": 97, "local_accuracy": 0.02, "jev_accuracy": 0.2425, "difference_pp": -22.25, "paired_cluster_bootstrap95_pp": [ -26.50625, -18.0 ], "local_only_correct": 7, "jev_only_correct": 96, "both_correct": 1, "both_wrong": 296, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb": { "n": 28, "local_correct": 25, "jev_correct": 28, "local_accuracy": 0.8928571428571429, "jev_accuracy": 1.0, "difference_pp": -10.714285714285714, "paired_cluster_bootstrap95_pp": [ -21.517857142857135, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 25, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 25, "jev_correct": 28, "local_accuracy": 0.8928571428571429, "jev_accuracy": 1.0, "difference_pp": -10.714285714285714, "paired_cluster_bootstrap95_pp": [ -21.517857142857135, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 25, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 33, "jev_correct": 40, "local_accuracy": 0.825, "jev_accuracy": 1.0, "difference_pp": -17.5, "paired_cluster_bootstrap95_pp": [ -30.0, -7.5 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 33, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 34, "jev_correct": 40, "local_accuracy": 0.85, "jev_accuracy": 1.0, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -27.5, -5.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 34, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 27, "jev_correct": 39, "local_accuracy": 0.675, "jev_accuracy": 0.975, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -45.0, -17.5 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 27, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 26, "jev_correct": 39, "local_accuracy": 0.65, "jev_accuracy": 0.975, "difference_pp": -32.5, "paired_cluster_bootstrap95_pp": [ -47.5, -20.0 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 26, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 211, "jev_correct": 237, "local_accuracy": 0.5275, "jev_accuracy": 0.5925, "difference_pp": -6.5, "paired_cluster_bootstrap95_pp": [ -11.5, -1.5 ], "local_only_correct": 40, "jev_only_correct": 66, "both_correct": 171, "both_wrong": 123, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 254, "jev_correct": 277, "local_accuracy": 0.8466666666666667, "jev_accuracy": 0.9233333333333333, "difference_pp": -7.666666666666666, "paired_cluster_bootstrap95_pp": [ -12.0, -3.6666666666666665 ], "local_only_correct": 11, "jev_only_correct": 34, "both_correct": 243, "both_wrong": 12, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 370, "jev_correct": 377, "local_accuracy": 0.925, "jev_accuracy": 0.9425, "difference_pp": -1.7500000000000002, "paired_cluster_bootstrap95_pp": [ -5.0, 1.25 ], "local_only_correct": 16, "jev_only_correct": 23, "both_correct": 354, "both_wrong": 7, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 316, "jev_correct": 346, "local_accuracy": 0.79, "jev_accuracy": 0.865, "difference_pp": -7.5, "paired_cluster_bootstrap95_pp": [ -10.75, -4.25 ], "local_only_correct": 8, "jev_only_correct": 38, "both_correct": 308, "both_wrong": 46, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 306, "jev_correct": 360, "local_accuracy": 0.765, "jev_accuracy": 0.9, "difference_pp": -13.5, "paired_cluster_bootstrap95_pp": [ -18.0, -8.75 ], "local_only_correct": 23, "jev_only_correct": 77, "both_correct": 283, "both_wrong": 17, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 230, "jev_correct": 225, "local_accuracy": 0.7666666666666667, "jev_accuracy": 0.75, "difference_pp": 1.6666666666666667, "paired_cluster_bootstrap95_pp": [ -3.3416666666666663, 7.0 ], "local_only_correct": 33, "jev_only_correct": 28, "both_correct": 197, "both_wrong": 42, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 300, "jev_correct": 393, "local_accuracy": 0.75, "jev_accuracy": 0.9825, "difference_pp": -23.25, "paired_cluster_bootstrap95_pp": [ -27.50625, -19.0 ], "local_only_correct": 1, "jev_only_correct": 94, "both_correct": 299, "both_wrong": 6, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 280, "jev_correct": 265, "local_accuracy": 0.7, "jev_accuracy": 0.6625, "difference_pp": 3.75, "paired_cluster_bootstrap95_pp": [ -1.25, 8.25 ], "local_only_correct": 54, "jev_only_correct": 39, "both_correct": 226, "both_wrong": 81, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 65, "jev_correct": 75, "local_accuracy": 0.65, "jev_accuracy": 0.75, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -18.0, -2.0 ], "local_only_correct": 4, "jev_only_correct": 14, "both_correct": 61, "both_wrong": 21, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 54, "jev_correct": 56, "local_accuracy": 0.54, "jev_accuracy": 0.56, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -10.0, 6.0 ], "local_only_correct": 7, "jev_only_correct": 9, "both_correct": 47, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 44, "jev_correct": 49, "local_accuracy": 0.44, "jev_accuracy": 0.49, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -14.0, 3.0249999999998636 ], "local_only_correct": 7, "jev_only_correct": 12, "both_correct": 37, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 41, "jev_correct": 48, "local_accuracy": 0.41, "jev_accuracy": 0.48, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -17.0, 3.0 ], "local_only_correct": 10, "jev_only_correct": 17, "both_correct": 31, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 76, "jev_correct": 79, "local_accuracy": 0.76, "jev_accuracy": 0.79, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -9.0, 3.0 ], "local_only_correct": 3, "jev_only_correct": 6, "both_correct": 73, "both_wrong": 18, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 5, "jev_correct": 6, "local_accuracy": 0.05, "jev_accuracy": 0.06, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -5.024999999999999, 3.0 ], "local_only_correct": 2, "jev_only_correct": 3, "both_correct": 3, "both_wrong": 92, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 56, "jev_correct": 64, "local_accuracy": 0.56, "jev_accuracy": 0.64, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -17.0, 1.0 ], "local_only_correct": 7, "jev_only_correct": 15, "both_correct": 49, "both_wrong": 29, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 35, "jev_correct": 37, "local_accuracy": 0.35, "jev_accuracy": 0.37, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -13.0, 9.0 ], "local_only_correct": 15, "jev_only_correct": 17, "both_correct": 20, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 4, "jev_correct": 16, "local_accuracy": 0.04, "jev_accuracy": 0.16, "difference_pp": -12.0, "paired_cluster_bootstrap95_pp": [ -19.0, -6.0 ], "local_only_correct": 1, "jev_only_correct": 13, "both_correct": 3, "both_wrong": 83, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 11, "jev_correct": 34, "local_accuracy": 0.11, "jev_accuracy": 0.34, "difference_pp": -23.0, "paired_cluster_bootstrap95_pp": [ -33.0, -13.0 ], "local_only_correct": 3, "jev_only_correct": 26, "both_correct": 8, "both_wrong": 63, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 69, "jev_correct": 83, "local_accuracy": 0.69, "jev_accuracy": 0.83, "difference_pp": -14.000000000000002, "paired_cluster_bootstrap95_pp": [ -22.0, -7.0 ], "local_only_correct": 1, "jev_only_correct": 15, "both_correct": 68, "both_wrong": 16, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 19, "jev_correct": 21, "local_accuracy": 0.19, "jev_accuracy": 0.21, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -12.0, 8.0 ], "local_only_correct": 13, "jev_only_correct": 15, "both_correct": 6, "both_wrong": 66, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 41, "jev_correct": 51, "local_accuracy": 0.41, "jev_accuracy": 0.51, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -22.0, 1.0249999999998636 ], "local_only_correct": 12, "jev_only_correct": 22, "both_correct": 29, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 58, "jev_correct": 65, "local_accuracy": 0.58, "jev_accuracy": 0.65, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -15.0, 2.0 ], "local_only_correct": 6, "jev_only_correct": 13, "both_correct": 52, "both_wrong": 29, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 39, "jev_correct": 39, "local_accuracy": 0.39, "jev_accuracy": 0.39, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -10.0, 10.0 ], "local_only_correct": 13, "jev_only_correct": 13, "both_correct": 26, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 5, "jev_correct": 9, "local_accuracy": 0.05, "jev_accuracy": 0.09, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -11.0, 3.0 ], "local_only_correct": 4, "jev_only_correct": 8, "both_correct": 1, "both_wrong": 87, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 67, "jev_correct": 59, "local_accuracy": 0.67, "jev_accuracy": 0.59, "difference_pp": 8.0, "paired_cluster_bootstrap95_pp": [ 1.0, 15.0 ], "local_only_correct": 11, "jev_only_correct": 3, "both_correct": 56, "both_wrong": 30, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 82, "jev_correct": 91, "local_accuracy": 0.82, "jev_accuracy": 0.91, "difference_pp": -9.0, "paired_cluster_bootstrap95_pp": [ -15.0, -3.0 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 81, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 84, "jev_correct": 88, "local_accuracy": 0.84, "jev_accuracy": 0.88, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -11.0, 3.0 ], "local_only_correct": 5, "jev_only_correct": 9, "both_correct": 79, "both_wrong": 7, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 49, "jev_correct": 48, "local_accuracy": 0.49, "jev_accuracy": 0.48, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -10.0, 12.0 ], "local_only_correct": 15, "jev_only_correct": 14, "both_correct": 34, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 1989, "jev_correct": 2550, "local_accuracy": 0.6746947082767978, "jev_accuracy": 0.8649932157394844, "difference_pp": -19.029850746268657, "paired_cluster_bootstrap95_pp": [ -20.658073270013567, -17.469470827679782 ], "local_only_correct": 68, "jev_only_correct": 629, "both_correct": 1921, "both_wrong": 330, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 1551, "jev_correct": 2412, "local_accuracy": 0.5261194029850746, "jev_accuracy": 0.8181818181818182, "difference_pp": -29.206241519674354, "paired_cluster_bootstrap95_pp": [ -31.10583446404342, -27.577170963365 ], "local_only_correct": 72, "jev_only_correct": 933, "both_correct": 1479, "both_wrong": 464, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 626, "jev_correct": 843, "local_accuracy": 0.5336743393009378, "jev_accuracy": 0.7186700767263428, "difference_pp": -18.499573742540495, "paired_cluster_bootstrap95_pp": [ -21.99869191777271, -15.202581178300605 ], "local_only_correct": 101, "jev_only_correct": 318, "both_correct": 525, "both_wrong": 229, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "decision2-kai-06b": { "fresh/news": { "n": 200, "local_correct": 167, "jev_correct": 178, "local_accuracy": 0.835, "jev_accuracy": 0.89, "difference_pp": -5.5, "paired_cluster_bootstrap95_pp": [ -9.5, -1.5 ], "local_only_correct": 3, "jev_only_correct": 14, "both_correct": 164, "both_wrong": 19, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 114, "jev_correct": 120, "local_accuracy": 0.57, "jev_accuracy": 0.6, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -9.5, 3.5 ], "local_only_correct": 19, "jev_only_correct": 25, "both_correct": 95, "both_wrong": 61, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 109, "jev_correct": 196, "local_accuracy": 0.545, "jev_accuracy": 0.98, "difference_pp": -43.5, "paired_cluster_bootstrap95_pp": [ -50.5, -36.5 ], "local_only_correct": 0, "jev_only_correct": 87, "both_correct": 109, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 137, "jev_correct": 179, "local_accuracy": 0.685, "jev_accuracy": 0.895, "difference_pp": -21.0, "paired_cluster_bootstrap95_pp": [ -27.5, -15.0 ], "local_only_correct": 5, "jev_only_correct": 47, "both_correct": 132, "both_wrong": 16, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 7, "jev_correct": 11, "local_accuracy": 0.5833333333333334, "jev_accuracy": 0.9166666666666666, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -58.333333333333336, -8.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 7, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 2, "jev_correct": 10, "local_accuracy": 0.16666666666666666, "jev_accuracy": 0.8333333333333334, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -100.0, -25.0 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 1, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 11, "jev_correct": 12, "local_accuracy": 0.9166666666666666, "jev_accuracy": 1.0, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 11, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 11, "jev_correct": 12, "local_accuracy": 0.9166666666666666, "jev_accuracy": 1.0, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 11, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 83, "jev_correct": 91, "local_accuracy": 0.83, "jev_accuracy": 0.91, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -14.0, -3.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 83, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 53, "jev_correct": 48, "local_accuracy": 0.53, "jev_accuracy": 0.48, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -3.0, 13.0 ], "local_only_correct": 11, "jev_only_correct": 6, "both_correct": 42, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 81, "jev_correct": 89, "local_accuracy": 0.81, "jev_accuracy": 0.89, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -15.0, -2.0 ], "local_only_correct": 2, "jev_only_correct": 10, "both_correct": 79, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 47, "jev_correct": 48, "local_accuracy": 0.9791666666666666, "jev_accuracy": 1.0, "difference_pp": -2.083333333333333, "paired_cluster_bootstrap95_pp": [ -6.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 47, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/hard": { "n": 111, "local_correct": 44, "jev_correct": 81, "local_accuracy": 0.3963963963963964, "jev_accuracy": 0.7297297297297297, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -44.14414414414414, -22.52252252252252 ], "local_only_correct": 4, "jev_only_correct": 41, "both_correct": 40, "both_wrong": 26, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 61, "jev_correct": 70, "local_accuracy": 0.8472222222222222, "jev_accuracy": 0.9722222222222222, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -20.833333333333332, -5.555555555555555 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 60, "both_wrong": 1, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8421052631578947, "jev_accuracy": 0.9473684210526315, "difference_pp": -10.526315789473683, "paired_cluster_bootstrap95_pp": [ -26.31578947368421, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 11, "jev_correct": 16, "local_accuracy": 0.6875, "jev_accuracy": 1.0, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 11, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 14, "jev_correct": 20, "local_accuracy": 0.6363636363636364, "jev_accuracy": 0.9090909090909091, "difference_pp": -27.27272727272727, "paired_cluster_bootstrap95_pp": [ -54.54545454545455, 0.0 ], "local_only_correct": 2, "jev_only_correct": 8, "both_correct": 12, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 11, "jev_correct": 18, "local_accuracy": 0.6111111111111112, "jev_accuracy": 1.0, "difference_pp": -38.88888888888889, "paired_cluster_bootstrap95_pp": [ -61.111111111111114, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 11, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.8823529411764706, "jev_accuracy": 0.9411764705882353, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -23.529411764705884, 11.764705882352942 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -60.0, -13.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8125, "jev_accuracy": 0.9375, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 12, "jev_correct": 18, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 12, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 15, "jev_correct": 21, "local_accuracy": 0.6818181818181818, "jev_accuracy": 0.9545454545454546, "difference_pp": -27.27272727272727, "paired_cluster_bootstrap95_pp": [ -45.45454545454545, -9.090909090909092 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 15, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 17, "jev_correct": 19, "local_accuracy": 0.8947368421052632, "jev_accuracy": 1.0, "difference_pp": -10.526315789473683, "paired_cluster_bootstrap95_pp": [ -26.31578947368421, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 17, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, -6.666666666666667 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9444444444444444, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4375, "jev_accuracy": 0.9375, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 7, "jev_correct": 17, "local_accuracy": 0.3888888888888889, "jev_accuracy": 0.9444444444444444, "difference_pp": -55.55555555555556, "paired_cluster_bootstrap95_pp": [ -77.77777777777777, -33.333333333333336 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 7, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7, "jev_accuracy": 0.9, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, -5.0 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 4, "jev_correct": 18, "local_accuracy": 0.19047619047619047, "jev_accuracy": 0.8571428571428571, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -85.71428571428571, -47.61904761904762 ], "local_only_correct": 0, "jev_only_correct": 14, "both_correct": 4, "both_wrong": 3, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 10, "jev_correct": 16, "local_accuracy": 0.625, "jev_accuracy": 1.0, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 10, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 9, "jev_correct": 14, "local_accuracy": 0.6428571428571429, "jev_accuracy": 1.0, "difference_pp": -35.714285714285715, "paired_cluster_bootstrap95_pp": [ -57.142857142857146, -14.285714285714286 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 9, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.6875, "jev_accuracy": 0.875, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 8, "jev_correct": 14, "local_accuracy": 0.5, "jev_accuracy": 0.875, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -68.75, -6.25 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 7, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 6, "jev_correct": 17, "local_accuracy": 0.35294117647058826, "jev_accuracy": 1.0, "difference_pp": -64.70588235294117, "paired_cluster_bootstrap95_pp": [ -88.23529411764706, -41.1764705882353 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 6, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7777777777777778, "jev_accuracy": 1.0, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.4444444444444444, "jev_accuracy": 0.9444444444444444, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -72.22222222222223, -27.77777777777778 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4375, "jev_accuracy": 0.9375, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5333333333333333, "jev_accuracy": 1.0, "difference_pp": -46.666666666666664, "paired_cluster_bootstrap95_pp": [ -73.33333333333333, -20.0 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.5789473684210527, "jev_accuracy": 0.8947368421052632, "difference_pp": -31.57894736842105, "paired_cluster_bootstrap95_pp": [ -57.89473684210526, -5.2631578947368425 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 9, "jev_correct": 19, "local_accuracy": 0.45, "jev_accuracy": 0.95, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -70.0, -30.0 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 9, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 8, "jev_correct": 16, "local_accuracy": 0.4, "jev_accuracy": 0.8, "difference_pp": -40.0, "paired_cluster_bootstrap95_pp": [ -60.0, -20.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 8, "both_wrong": 4, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 18, "jev_correct": 20, "local_accuracy": 0.8181818181818182, "jev_accuracy": 0.9090909090909091, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -27.272727272727273, 9.090909090909092 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 17, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 420, "jev_correct": 629, "local_accuracy": 0.4666666666666667, "jev_accuracy": 0.6988888888888889, "difference_pp": -23.22222222222222, "paired_cluster_bootstrap95_pp": [ -28.2945164848019, -18.863764098953727 ], "local_only_correct": 65, "jev_only_correct": 274, "both_correct": 355, "both_wrong": 206, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 270, "jev_correct": 465, "local_accuracy": 0.47038327526132406, "jev_accuracy": 0.8101045296167247, "difference_pp": -33.972125435540065, "paired_cluster_bootstrap95_pp": [ -38.153310104529616, -29.61672473867596 ], "local_only_correct": 22, "jev_only_correct": 217, "both_correct": 248, "both_wrong": 87, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 173, "jev_correct": 490, "local_accuracy": 0.24714285714285714, "jev_accuracy": 0.7, "difference_pp": -45.285714285714285, "paired_cluster_bootstrap95_pp": [ -49.57197855730453, -40.94745880814958 ], "local_only_correct": 30, "jev_only_correct": 347, "both_correct": 143, "both_wrong": 180, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 15, "jev_correct": 97, "local_accuracy": 0.0375, "jev_accuracy": 0.2425, "difference_pp": -20.5, "paired_cluster_bootstrap95_pp": [ -25.00625, -15.75 ], "local_only_correct": 12, "jev_only_correct": 94, "both_correct": 3, "both_wrong": 291, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb": { "n": 28, "local_correct": 26, "jev_correct": 28, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -17.857142857142858, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 26, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 35, "jev_correct": 40, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -25.0, -2.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 35, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 35, "jev_correct": 40, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -25.0, -2.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 35, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 31, "jev_correct": 39, "local_accuracy": 0.775, "jev_accuracy": 0.975, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -35.0, -7.5 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 30, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 31, "jev_correct": 39, "local_accuracy": 0.775, "jev_accuracy": 0.975, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -35.0, -7.5 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 30, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 220, "jev_correct": 237, "local_accuracy": 0.55, "jev_accuracy": 0.5925, "difference_pp": -4.25, "paired_cluster_bootstrap95_pp": [ -8.75625, 0.5 ], "local_only_correct": 36, "jev_only_correct": 53, "both_correct": 184, "both_wrong": 127, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 257, "jev_correct": 277, "local_accuracy": 0.8566666666666667, "jev_accuracy": 0.9233333333333333, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -10.666666666666666, -3.0 ], "local_only_correct": 8, "jev_only_correct": 28, "both_correct": 249, "both_wrong": 15, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 350, "jev_correct": 377, "local_accuracy": 0.875, "jev_accuracy": 0.9425, "difference_pp": -6.75, "paired_cluster_bootstrap95_pp": [ -10.5, -2.75 ], "local_only_correct": 17, "jev_only_correct": 44, "both_correct": 333, "both_wrong": 6, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 335, "jev_correct": 346, "local_accuracy": 0.8375, "jev_accuracy": 0.865, "difference_pp": -2.75, "paired_cluster_bootstrap95_pp": [ -5.5, -0.25 ], "local_only_correct": 8, "jev_only_correct": 19, "both_correct": 327, "both_wrong": 46, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 286, "jev_correct": 360, "local_accuracy": 0.715, "jev_accuracy": 0.9, "difference_pp": -18.5, "paired_cluster_bootstrap95_pp": [ -23.25, -14.0 ], "local_only_correct": 16, "jev_only_correct": 90, "both_correct": 270, "both_wrong": 24, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 199, "jev_correct": 225, "local_accuracy": 0.6633333333333333, "jev_accuracy": 0.75, "difference_pp": -8.666666666666668, "paired_cluster_bootstrap95_pp": [ -13.333333333333334, -3.6666666666666665 ], "local_only_correct": 15, "jev_only_correct": 41, "both_correct": 184, "both_wrong": 60, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 192, "jev_correct": 393, "local_accuracy": 0.48, "jev_accuracy": 0.9825, "difference_pp": -50.24999999999999, "paired_cluster_bootstrap95_pp": [ -55.25, -45.243750000000034 ], "local_only_correct": 2, "jev_only_correct": 203, "both_correct": 190, "both_wrong": 5, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 263, "jev_correct": 265, "local_accuracy": 0.6575, "jev_accuracy": 0.6625, "difference_pp": -0.5, "paired_cluster_bootstrap95_pp": [ -4.75, 3.5 ], "local_only_correct": 34, "jev_only_correct": 36, "both_correct": 229, "both_wrong": 101, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 58, "jev_correct": 75, "local_accuracy": 0.58, "jev_accuracy": 0.75, "difference_pp": -17.0, "paired_cluster_bootstrap95_pp": [ -28.0, -6.0 ], "local_only_correct": 10, "jev_only_correct": 27, "both_correct": 48, "both_wrong": 15, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 55, "jev_correct": 56, "local_accuracy": 0.55, "jev_accuracy": 0.56, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -9.0, 8.0 ], "local_only_correct": 9, "jev_only_correct": 10, "both_correct": 46, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 49, "jev_correct": 49, "local_accuracy": 0.49, "jev_accuracy": 0.49, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -9.0, 8.024999999999864 ], "local_only_correct": 9, "jev_only_correct": 9, "both_correct": 40, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 41, "jev_correct": 48, "local_accuracy": 0.41, "jev_accuracy": 0.48, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -16.0, 2.0 ], "local_only_correct": 9, "jev_only_correct": 16, "both_correct": 32, "both_wrong": 43, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 80, "jev_correct": 79, "local_accuracy": 0.8, "jev_accuracy": 0.79, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 7.0 ], "local_only_correct": 5, "jev_only_correct": 4, "both_correct": 75, "both_wrong": 16, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 8, "jev_correct": 6, "local_accuracy": 0.08, "jev_accuracy": 0.06, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -3.0, 7.0 ], "local_only_correct": 4, "jev_only_correct": 2, "both_correct": 4, "both_wrong": 90, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 57, "jev_correct": 64, "local_accuracy": 0.57, "jev_accuracy": 0.64, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -16.0, 1.0249999999998636 ], "local_only_correct": 6, "jev_only_correct": 13, "both_correct": 51, "both_wrong": 30, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 42, "jev_correct": 37, "local_accuracy": 0.42, "jev_accuracy": 0.37, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -5.0, 15.0 ], "local_only_correct": 15, "jev_only_correct": 10, "both_correct": 27, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 4, "jev_correct": 16, "local_accuracy": 0.04, "jev_accuracy": 0.16, "difference_pp": -12.0, "paired_cluster_bootstrap95_pp": [ -20.0, -5.0 ], "local_only_correct": 2, "jev_only_correct": 14, "both_correct": 2, "both_wrong": 82, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 13, "jev_correct": 34, "local_accuracy": 0.13, "jev_accuracy": 0.34, "difference_pp": -21.0, "paired_cluster_bootstrap95_pp": [ -30.0, -12.0 ], "local_only_correct": 2, "jev_only_correct": 23, "both_correct": 11, "both_wrong": 64, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 68, "jev_correct": 83, "local_accuracy": 0.68, "jev_accuracy": 0.83, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -24.0, -6.0 ], "local_only_correct": 3, "jev_only_correct": 18, "both_correct": 65, "both_wrong": 14, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 15, "jev_correct": 21, "local_accuracy": 0.15, "jev_accuracy": 0.21, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -16.0, 4.0 ], "local_only_correct": 10, "jev_only_correct": 16, "both_correct": 5, "both_wrong": 69, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 40, "jev_correct": 51, "local_accuracy": 0.4, "jev_accuracy": 0.51, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -22.0, 0.0 ], "local_only_correct": 11, "jev_only_correct": 22, "both_correct": 29, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 60, "jev_correct": 65, "local_accuracy": 0.6, "jev_accuracy": 0.65, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.0, 2.0 ], "local_only_correct": 4, "jev_only_correct": 9, "both_correct": 56, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 40, "jev_correct": 39, "local_accuracy": 0.4, "jev_accuracy": 0.39, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -8.0, 10.0 ], "local_only_correct": 10, "jev_only_correct": 9, "both_correct": 30, "both_wrong": 51, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 4, "jev_correct": 9, "local_accuracy": 0.04, "jev_accuracy": 0.09, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.0, 1.0 ], "local_only_correct": 3, "jev_only_correct": 8, "both_correct": 1, "both_wrong": 88, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 60, "jev_correct": 59, "local_accuracy": 0.6, "jev_accuracy": 0.59, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -6.0, 8.024999999999864 ], "local_only_correct": 8, "jev_only_correct": 7, "both_correct": 52, "both_wrong": 33, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 82, "jev_correct": 91, "local_accuracy": 0.82, "jev_accuracy": 0.91, "difference_pp": -9.0, "paired_cluster_bootstrap95_pp": [ -15.0, -4.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 82, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 81, "jev_correct": 88, "local_accuracy": 0.81, "jev_accuracy": 0.88, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -13.0, -2.0 ], "local_only_correct": 1, "jev_only_correct": 8, "both_correct": 80, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 53, "jev_correct": 48, "local_accuracy": 0.53, "jev_accuracy": 0.48, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -5.0, 15.0 ], "local_only_correct": 15, "jev_only_correct": 10, "both_correct": 38, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2149, "jev_correct": 2550, "local_accuracy": 0.7289687924016283, "jev_accuracy": 0.8649932157394844, "difference_pp": -13.602442333785616, "paired_cluster_bootstrap95_pp": [ -15.027985074626866, -12.109905020352782 ], "local_only_correct": 69, "jev_only_correct": 470, "both_correct": 2080, "both_wrong": 329, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 1614, "jev_correct": 2412, "local_accuracy": 0.5474898236092266, "jev_accuracy": 0.8181818181818182, "difference_pp": -27.069199457259156, "paired_cluster_bootstrap95_pp": [ -28.900949796472183, -25.27137042062415 ], "local_only_correct": 76, "jev_only_correct": 874, "both_correct": 1538, "both_wrong": 460, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 677, "jev_correct": 843, "local_accuracy": 0.577152600170503, "jev_accuracy": 0.7186700767263428, "difference_pp": -14.151747655583973, "paired_cluster_bootstrap95_pp": [ -17.558841703179418, -10.562214812214815 ], "local_only_correct": 116, "jev_only_correct": 282, "both_correct": 561, "both_wrong": 214, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "decision2-nox-4b": { "fresh/news": { "n": 200, "local_correct": 174, "jev_correct": 178, "local_accuracy": 0.87, "jev_accuracy": 0.89, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -5.0, 0.5 ], "local_only_correct": 2, "jev_only_correct": 6, "both_correct": 172, "both_wrong": 20, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 118, "jev_correct": 120, "local_accuracy": 0.59, "jev_accuracy": 0.6, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -6.0, 4.5 ], "local_only_correct": 14, "jev_only_correct": 16, "both_correct": 104, "both_wrong": 66, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 186, "jev_correct": 196, "local_accuracy": 0.93, "jev_accuracy": 0.98, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -8.5, -2.0 ], "local_only_correct": 1, "jev_only_correct": 11, "both_correct": 185, "both_wrong": 3, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 185, "jev_correct": 179, "local_accuracy": 0.925, "jev_accuracy": 0.895, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -1.5, 7.5 ], "local_only_correct": 13, "jev_only_correct": 7, "both_correct": 172, "both_wrong": 8, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 10, "jev_correct": 11, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9166666666666666, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 10, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 10, "jev_correct": 10, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.8333333333333334, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 10, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/en": { "n": 12, "local_correct": 11, "jev_correct": 12, "local_accuracy": 0.9166666666666666, "jev_accuracy": 1.0, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 11, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 85, "jev_correct": 91, "local_accuracy": 0.85, "jev_accuracy": 0.91, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -11.0, -2.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 85, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 51, "jev_correct": 48, "local_accuracy": 0.51, "jev_accuracy": 0.48, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -6.0, 12.0 ], "local_only_correct": 12, "jev_only_correct": 9, "both_correct": 39, "both_wrong": 40, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 86, "jev_correct": 89, "local_accuracy": 0.86, "jev_accuracy": 0.89, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -8.0, 2.0 ], "local_only_correct": 2, "jev_only_correct": 5, "both_correct": 84, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 69, "jev_correct": 81, "local_accuracy": 0.6216216216216216, "jev_accuracy": 0.7297297297297297, "difference_pp": -10.81081081081081, "paired_cluster_bootstrap95_pp": [ -18.01801801801802, -3.6036036036036037 ], "local_only_correct": 3, "jev_only_correct": 15, "both_correct": 66, "both_wrong": 27, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 68, "jev_correct": 70, "local_accuracy": 0.9444444444444444, "jev_accuracy": 0.9722222222222222, "difference_pp": -2.7777777777777777, "paired_cluster_bootstrap95_pp": [ -12.5, 4.166666666666667 ], "local_only_correct": 2, "jev_only_correct": 4, "both_correct": 66, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.9473684210526315, "jev_accuracy": 0.9473684210526315, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 13.636363636363637 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.8823529411764706, "jev_accuracy": 0.9411764705882353, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7777777777777778, "jev_accuracy": 1.0, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 20, "jev_correct": 21, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9545454545454546, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 20, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 17, "jev_correct": 17, "local_accuracy": 0.9444444444444444, "jev_accuracy": 0.9444444444444444, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/formality_level": { "n": 16, "local_correct": 9, "jev_correct": 15, "local_accuracy": 0.5625, "jev_accuracy": 0.9375, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 9, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 17, "jev_correct": 17, "local_accuracy": 0.9444444444444444, "jev_accuracy": 0.9444444444444444, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 19, "jev_correct": 18, "local_accuracy": 0.95, "jev_accuracy": 0.9, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ 0.0, 15.0 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 12, "jev_correct": 18, "local_accuracy": 0.5714285714285714, "jev_accuracy": 0.8571428571428571, "difference_pp": -28.57142857142857, "paired_cluster_bootstrap95_pp": [ -52.38095238095238, -4.761904761904762 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 11, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.8571428571428571, "jev_accuracy": 1.0, "difference_pp": -14.285714285714285, "paired_cluster_bootstrap95_pp": [ -35.714285714285715, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 14, "jev_correct": 14, "local_accuracy": 0.875, "jev_accuracy": 0.875, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -18.75, 18.75 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 14, "jev_correct": 14, "local_accuracy": 0.875, "jev_accuracy": 0.875, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -18.75, 18.75 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9444444444444444, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.6875, "jev_accuracy": 0.9375, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -60.0, -13.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7368421052631579, "jev_accuracy": 0.8947368421052632, "difference_pp": -15.789473684210526, "paired_cluster_bootstrap95_pp": [ -31.57894736842105, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 2, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 16, "jev_correct": 19, "local_accuracy": 0.8, "jev_accuracy": 0.95, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -35.0, 5.0 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 15, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 20, "jev_correct": 20, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 17, "jev_correct": 16, "local_accuracy": 0.85, "jev_accuracy": 0.8, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -10.0, 20.0 ], "local_only_correct": 2, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 13.636363636363637 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 588, "jev_correct": 629, "local_accuracy": 0.6533333333333333, "jev_accuracy": 0.6988888888888889, "difference_pp": -4.555555555555555, "paired_cluster_bootstrap95_pp": [ -7.638413910067002, -1.767105890937667 ], "local_only_correct": 49, "jev_only_correct": 90, "both_correct": 539, "both_wrong": 222, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 403, "jev_correct": 465, "local_accuracy": 0.7020905923344948, "jev_accuracy": 0.8101045296167247, "difference_pp": -10.801393728222997, "paired_cluster_bootstrap95_pp": [ -14.111498257839722, -7.4869337979094315 ], "local_only_correct": 18, "jev_only_correct": 80, "both_correct": 385, "both_wrong": 91, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 352, "jev_correct": 490, "local_accuracy": 0.5028571428571429, "jev_accuracy": 0.7, "difference_pp": -19.714285714285715, "paired_cluster_bootstrap95_pp": [ -23.60599567126185, -15.72835497835498 ], "local_only_correct": 47, "jev_only_correct": 185, "both_correct": 305, "both_wrong": 163, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 76, "jev_correct": 97, "local_accuracy": 0.19, "jev_accuracy": 0.2425, "difference_pp": -5.25, "paired_cluster_bootstrap95_pp": [ -10.5, 0.0 ], "local_only_correct": 51, "jev_only_correct": 72, "both_correct": 25, "both_wrong": 252, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en/reversed": { "n": 40, "local_correct": 38, "jev_correct": 40, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 38, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 38, "jev_correct": 40, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 38, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 34, "jev_correct": 39, "local_accuracy": 0.85, "jev_accuracy": 0.975, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -22.562499999999996, -2.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 34, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 35, "jev_correct": 39, "local_accuracy": 0.875, "jev_accuracy": 0.975, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -20.0, -2.5 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 35, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 232, "jev_correct": 237, "local_accuracy": 0.58, "jev_accuracy": 0.5925, "difference_pp": -1.25, "paired_cluster_bootstrap95_pp": [ -5.0, 2.75 ], "local_only_correct": 29, "jev_only_correct": 34, "both_correct": 203, "both_wrong": 134, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 271, "jev_correct": 277, "local_accuracy": 0.9033333333333333, "jev_accuracy": 0.9233333333333333, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -5.0, 1.0 ], "local_only_correct": 7, "jev_only_correct": 13, "both_correct": 264, "both_wrong": 16, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 315, "jev_correct": 377, "local_accuracy": 0.7875, "jev_accuracy": 0.9425, "difference_pp": -15.5, "paired_cluster_bootstrap95_pp": [ -19.25, -11.75 ], "local_only_correct": 4, "jev_only_correct": 66, "both_correct": 311, "both_wrong": 19, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 342, "jev_correct": 346, "local_accuracy": 0.855, "jev_accuracy": 0.865, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -3.25, 1.25 ], "local_only_correct": 8, "jev_only_correct": 12, "both_correct": 334, "both_wrong": 46, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 368, "jev_correct": 360, "local_accuracy": 0.92, "jev_accuracy": 0.9, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -0.75, 5.0 ], "local_only_correct": 20, "jev_only_correct": 12, "both_correct": 348, "both_wrong": 20, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 225, "jev_correct": 225, "local_accuracy": 0.75, "jev_accuracy": 0.75, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -3.0, 2.6666666666666665 ], "local_only_correct": 10, "jev_only_correct": 10, "both_correct": 215, "both_wrong": 65, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 374, "jev_correct": 393, "local_accuracy": 0.935, "jev_accuracy": 0.9825, "difference_pp": -4.75, "paired_cluster_bootstrap95_pp": [ -7.0, -2.75 ], "local_only_correct": 1, "jev_only_correct": 20, "both_correct": 373, "both_wrong": 6, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 336, "jev_correct": 265, "local_accuracy": 0.84, "jev_accuracy": 0.6625, "difference_pp": 17.75, "paired_cluster_bootstrap95_pp": [ 13.5, 22.0 ], "local_only_correct": 80, "jev_only_correct": 9, "both_correct": 256, "both_wrong": 55, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 81, "jev_correct": 75, "local_accuracy": 0.81, "jev_accuracy": 0.75, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ 0.0, 12.0 ], "local_only_correct": 8, "jev_only_correct": 2, "both_correct": 73, "both_wrong": 17, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 59, "jev_correct": 56, "local_accuracy": 0.59, "jev_accuracy": 0.56, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -4.0, 11.0 ], "local_only_correct": 9, "jev_only_correct": 6, "both_correct": 50, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 47, "jev_correct": 49, "local_accuracy": 0.47, "jev_accuracy": 0.49, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -7.0, 3.0 ], "local_only_correct": 2, "jev_only_correct": 4, "both_correct": 45, "both_wrong": 49, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 48, "jev_correct": 48, "local_accuracy": 0.48, "jev_accuracy": 0.48, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -8.0, 8.0 ], "local_only_correct": 8, "jev_only_correct": 8, "both_correct": 40, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 80, "jev_correct": 79, "local_accuracy": 0.8, "jev_accuracy": 0.79, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -2.0, 5.0 ], "local_only_correct": 2, "jev_only_correct": 1, "both_correct": 78, "both_wrong": 19, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 8, "jev_correct": 6, "local_accuracy": 0.08, "jev_accuracy": 0.06, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -3.0, 8.0 ], "local_only_correct": 5, "jev_only_correct": 3, "both_correct": 3, "both_wrong": 89, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 67, "jev_correct": 64, "local_accuracy": 0.67, "jev_accuracy": 0.64, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -4.0, 11.0 ], "local_only_correct": 9, "jev_only_correct": 6, "both_correct": 58, "both_wrong": 27, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 43, "jev_correct": 37, "local_accuracy": 0.43, "jev_accuracy": 0.37, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ -1.0, 13.0 ], "local_only_correct": 10, "jev_only_correct": 4, "both_correct": 33, "both_wrong": 53, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.13, "jev_accuracy": 0.16, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -10.0, 3.0 ], "local_only_correct": 4, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 80, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 42, "jev_correct": 34, "local_accuracy": 0.42, "jev_accuracy": 0.34, "difference_pp": 8.0, "paired_cluster_bootstrap95_pp": [ 1.0, 16.0 ], "local_only_correct": 11, "jev_only_correct": 3, "both_correct": 31, "both_wrong": 55, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 87, "jev_correct": 83, "local_accuracy": 0.87, "jev_accuracy": 0.83, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -2.0, 10.0 ], "local_only_correct": 6, "jev_only_correct": 2, "both_correct": 81, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 19, "jev_correct": 21, "local_accuracy": 0.19, "jev_accuracy": 0.21, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -11.0, 7.0 ], "local_only_correct": 10, "jev_only_correct": 12, "both_correct": 9, "both_wrong": 69, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 48, "jev_correct": 51, "local_accuracy": 0.48, "jev_accuracy": 0.51, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -11.0, 6.0 ], "local_only_correct": 7, "jev_only_correct": 10, "both_correct": 41, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 69, "jev_correct": 65, "local_accuracy": 0.69, "jev_accuracy": 0.65, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -1.0249999999999986, 10.0 ], "local_only_correct": 6, "jev_only_correct": 2, "both_correct": 63, "both_wrong": 29, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 44, "jev_correct": 39, "local_accuracy": 0.44, "jev_accuracy": 0.39, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -2.0, 12.0 ], "local_only_correct": 9, "jev_only_correct": 4, "both_correct": 35, "both_wrong": 52, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 12, "jev_correct": 9, "local_accuracy": 0.12, "jev_accuracy": 0.09, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -4.0, 10.0 ], "local_only_correct": 8, "jev_only_correct": 5, "both_correct": 4, "both_wrong": 83, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 67, "jev_correct": 59, "local_accuracy": 0.67, "jev_accuracy": 0.59, "difference_pp": 8.0, "paired_cluster_bootstrap95_pp": [ 2.0, 15.0 ], "local_only_correct": 10, "jev_only_correct": 2, "both_correct": 57, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 88, "jev_correct": 91, "local_accuracy": 0.88, "jev_accuracy": 0.91, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -7.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 88, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 87, "jev_correct": 88, "local_accuracy": 0.87, "jev_accuracy": 0.88, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 3.0 ], "local_only_correct": 2, "jev_only_correct": 3, "both_correct": 85, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 53, "jev_correct": 48, "local_accuracy": 0.53, "jev_accuracy": 0.48, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -5.0, 15.0 ], "local_only_correct": 15, "jev_only_correct": 10, "both_correct": 38, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2512, "jev_correct": 2550, "local_accuracy": 0.8521031207598372, "jev_accuracy": 0.8649932157394844, "difference_pp": -1.289009497964722, "paired_cluster_bootstrap95_pp": [ -2.2048846675712346, -0.33921302578018997 ], "local_only_correct": 81, "jev_only_correct": 119, "both_correct": 2431, "both_wrong": 317, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2367, "jev_correct": 2412, "local_accuracy": 0.8029172320217096, "jev_accuracy": 0.8181818181818182, "difference_pp": -1.5264586160108549, "paired_cluster_bootstrap95_pp": [ -2.611940298507463, -0.44097693351424694 ], "local_only_correct": 109, "jev_only_correct": 154, "both_correct": 2258, "both_wrong": 427, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 844, "jev_correct": 843, "local_accuracy": 0.7195225916453538, "jev_accuracy": 0.7186700767263428, "difference_pp": 0.08525149190110827, "paired_cluster_bootstrap95_pp": [ -2.8841713476308484, 3.1598436453905436 ], "local_only_correct": 134, "jev_only_correct": 133, "both_correct": 710, "both_wrong": 196, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "decision2-sol-2b": { "fresh/news": { "n": 200, "local_correct": 167, "jev_correct": 178, "local_accuracy": 0.835, "jev_accuracy": 0.89, "difference_pp": -5.5, "paired_cluster_bootstrap95_pp": [ -9.5, -2.0 ], "local_only_correct": 2, "jev_only_correct": 13, "both_correct": 165, "both_wrong": 20, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 58, "jev_correct": 120, "local_accuracy": 0.29, "jev_accuracy": 0.6, "difference_pp": -31.0, "paired_cluster_bootstrap95_pp": [ -39.5, -22.0 ], "local_only_correct": 19, "jev_only_correct": 81, "both_correct": 39, "both_wrong": 61, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 149, "jev_correct": 196, "local_accuracy": 0.745, "jev_accuracy": 0.98, "difference_pp": -23.5, "paired_cluster_bootstrap95_pp": [ -29.5, -18.0 ], "local_only_correct": 1, "jev_only_correct": 48, "both_correct": 148, "both_wrong": 3, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 172, "jev_correct": 179, "local_accuracy": 0.86, "jev_accuracy": 0.895, "difference_pp": -3.5000000000000004, "paired_cluster_bootstrap95_pp": [ -9.0, 2.0 ], "local_only_correct": 12, "jev_only_correct": 19, "both_correct": 160, "both_wrong": 9, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 7, "jev_correct": 11, "local_accuracy": 0.5833333333333334, "jev_accuracy": 0.9166666666666666, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -58.333333333333336, -8.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 7, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 7, "jev_correct": 10, "local_accuracy": 0.5833333333333334, "jev_accuracy": 0.8333333333333334, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 7, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 80, "jev_correct": 91, "local_accuracy": 0.8, "jev_accuracy": 0.91, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -18.0, -5.975000000000136 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 80, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 52, "jev_correct": 48, "local_accuracy": 0.52, "jev_accuracy": 0.48, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -5.0, 14.0 ], "local_only_correct": 14, "jev_only_correct": 10, "both_correct": 38, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 89, "jev_correct": 89, "local_accuracy": 0.89, "jev_accuracy": 0.89, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -5.024999999999999, 6.0 ], "local_only_correct": 4, "jev_only_correct": 4, "both_correct": 85, "both_wrong": 7, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 51, "jev_correct": 81, "local_accuracy": 0.4594594594594595, "jev_accuracy": 0.7297297297297297, "difference_pp": -27.027027027027028, "paired_cluster_bootstrap95_pp": [ -36.93693693693694, -17.117117117117118 ], "local_only_correct": 3, "jev_only_correct": 33, "both_correct": 48, "both_wrong": 27, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 67, "jev_correct": 70, "local_accuracy": 0.9305555555555556, "jev_accuracy": 0.9722222222222222, "difference_pp": -4.166666666666666, "paired_cluster_bootstrap95_pp": [ -11.11111111111111, 1.3888888888888888 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 66, "both_wrong": 1, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.8947368421052632, "jev_accuracy": 0.9473684210526315, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 15, "jev_correct": 20, "local_accuracy": 0.6818181818181818, "jev_accuracy": 0.9090909090909091, "difference_pp": -22.727272727272727, "paired_cluster_bootstrap95_pp": [ -45.45454545454545, 4.545454545454546 ], "local_only_correct": 2, "jev_only_correct": 7, "both_correct": 13, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 12, "jev_correct": 18, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 12, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 16, "jev_correct": 16, "local_accuracy": 0.9411764705882353, "jev_accuracy": 0.9411764705882353, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 17.647058823529413 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.7333333333333333, "jev_accuracy": 1.0, "difference_pp": -26.666666666666668, "paired_cluster_bootstrap95_pp": [ -46.666666666666664, -6.666666666666667 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 9, "jev_correct": 15, "local_accuracy": 0.5625, "jev_accuracy": 0.9375, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 9, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 12, "jev_correct": 18, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 12, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 20, "jev_correct": 21, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9545454545454546, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 20, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 17, "jev_correct": 17, "local_accuracy": 0.9444444444444444, "jev_accuracy": 0.9444444444444444, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 16.666666666666668 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5, "jev_accuracy": 0.9375, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7777777777777778, "jev_accuracy": 0.9444444444444444, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.9, "jev_accuracy": 0.9, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -15.0, 15.0 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 7, "jev_correct": 18, "local_accuracy": 0.3333333333333333, "jev_accuracy": 0.8571428571428571, "difference_pp": -52.38095238095239, "paired_cluster_bootstrap95_pp": [ -76.19047619047619, -23.80952380952381 ], "local_only_correct": 1, "jev_only_correct": 12, "both_correct": 6, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 13, "jev_correct": 14, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -21.428571428571427, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 13, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 10, "jev_correct": 14, "local_accuracy": 0.625, "jev_accuracy": 0.875, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 10, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.6875, "jev_accuracy": 0.875, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -43.75, 6.25 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.8125, "jev_accuracy": 1.0, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.7222222222222222, "jev_accuracy": 1.0, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9444444444444444, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -5.416666666667425 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5333333333333333, "jev_accuracy": 1.0, "difference_pp": -46.666666666666664, "paired_cluster_bootstrap95_pp": [ -73.33333333333333, -20.0 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.5789473684210527, "jev_accuracy": 0.8947368421052632, "difference_pp": -31.57894736842105, "paired_cluster_bootstrap95_pp": [ -57.89473684210526, -5.2631578947368425 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 12, "jev_correct": 19, "local_accuracy": 0.6, "jev_accuracy": 0.95, "difference_pp": -35.0, "paired_cluster_bootstrap95_pp": [ -55.0, -15.0 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 12, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.6, "jev_accuracy": 0.8, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, -5.0 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 4, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 16, "jev_correct": 20, "local_accuracy": 0.7272727272727273, "jev_accuracy": 0.9090909090909091, "difference_pp": -18.181818181818183, "paired_cluster_bootstrap95_pp": [ -36.36363636363637, -4.545454545454546 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 16, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 512, "jev_correct": 629, "local_accuracy": 0.5688888888888889, "jev_accuracy": 0.6988888888888889, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -17.810165493083602, -8.849917206081598 ], "local_only_correct": 68, "jev_only_correct": 185, "both_correct": 444, "both_wrong": 203, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 300, "jev_correct": 465, "local_accuracy": 0.5226480836236934, "jev_accuracy": 0.8101045296167247, "difference_pp": -28.745644599303137, "paired_cluster_bootstrap95_pp": [ -32.926829268292686, -24.390243902439025 ], "local_only_correct": 21, "jev_only_correct": 186, "both_correct": 279, "both_wrong": 88, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 161, "jev_correct": 490, "local_accuracy": 0.23, "jev_accuracy": 0.7, "difference_pp": -47.0, "paired_cluster_bootstrap95_pp": [ -51.06990014265335, -42.836879432624116 ], "local_only_correct": 21, "jev_only_correct": 350, "both_correct": 140, "both_wrong": 189, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 25, "jev_correct": 97, "local_accuracy": 0.0625, "jev_accuracy": 0.2425, "difference_pp": -18.0, "paired_cluster_bootstrap95_pp": [ -22.75, -13.5 ], "local_only_correct": 15, "jev_only_correct": 87, "both_correct": 10, "both_wrong": 288, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 31, "jev_correct": 40, "local_accuracy": 0.775, "jev_accuracy": 1.0, "difference_pp": -22.5, "paired_cluster_bootstrap95_pp": [ -35.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 31, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 35, "jev_correct": 40, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -22.5, -2.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 35, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 28, "jev_correct": 39, "local_accuracy": 0.7, "jev_accuracy": 0.975, "difference_pp": -27.500000000000004, "paired_cluster_bootstrap95_pp": [ -42.5, -15.0 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 28, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 30, "jev_correct": 39, "local_accuracy": 0.75, "jev_accuracy": 0.975, "difference_pp": -22.5, "paired_cluster_bootstrap95_pp": [ -37.5, -7.5 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 29, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 126, "jev_correct": 237, "local_accuracy": 0.315, "jev_accuracy": 0.5925, "difference_pp": -27.750000000000004, "paired_cluster_bootstrap95_pp": [ -33.25, -22.0 ], "local_only_correct": 33, "jev_only_correct": 144, "both_correct": 93, "both_wrong": 130, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 254, "jev_correct": 277, "local_accuracy": 0.8466666666666667, "jev_accuracy": 0.9233333333333333, "difference_pp": -7.666666666666666, "paired_cluster_bootstrap95_pp": [ -11.341666666666667, -3.6666666666666665 ], "local_only_correct": 7, "jev_only_correct": 30, "both_correct": 247, "both_wrong": 16, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 367, "jev_correct": 377, "local_accuracy": 0.9175, "jev_accuracy": 0.9425, "difference_pp": -2.5, "paired_cluster_bootstrap95_pp": [ -5.0, 0.0 ], "local_only_correct": 8, "jev_only_correct": 18, "both_correct": 359, "both_wrong": 15, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 328, "jev_correct": 346, "local_accuracy": 0.82, "jev_accuracy": 0.865, "difference_pp": -4.5, "paired_cluster_bootstrap95_pp": [ -7.25, -2.0 ], "local_only_correct": 6, "jev_only_correct": 24, "both_correct": 322, "both_wrong": 48, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 341, "jev_correct": 360, "local_accuracy": 0.8525, "jev_accuracy": 0.9, "difference_pp": -4.75, "paired_cluster_bootstrap95_pp": [ -9.0, -0.4937500000000341 ], "local_only_correct": 30, "jev_only_correct": 49, "both_correct": 311, "both_wrong": 10, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 217, "jev_correct": 225, "local_accuracy": 0.7233333333333334, "jev_accuracy": 0.75, "difference_pp": -2.666666666666667, "paired_cluster_bootstrap95_pp": [ -7.333333333333333, 2.008333333333288 ], "local_only_correct": 22, "jev_only_correct": 30, "both_correct": 195, "both_wrong": 53, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 320, "jev_correct": 393, "local_accuracy": 0.8, "jev_accuracy": 0.9825, "difference_pp": -18.25, "paired_cluster_bootstrap95_pp": [ -22.0, -14.5 ], "local_only_correct": 0, "jev_only_correct": 73, "both_correct": 320, "both_wrong": 7, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 316, "jev_correct": 265, "local_accuracy": 0.79, "jev_accuracy": 0.6625, "difference_pp": 12.75, "paired_cluster_bootstrap95_pp": [ 7.25, 18.25 ], "local_only_correct": 90, "jev_only_correct": 39, "both_correct": 226, "both_wrong": 45, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 75, "jev_correct": 75, "local_accuracy": 0.75, "jev_accuracy": 0.75, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -8.0, 8.0 ], "local_only_correct": 8, "jev_only_correct": 8, "both_correct": 67, "both_wrong": 17, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 55, "jev_correct": 56, "local_accuracy": 0.55, "jev_accuracy": 0.56, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -11.0, 8.0 ], "local_only_correct": 11, "jev_only_correct": 12, "both_correct": 44, "both_wrong": 33, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 50, "jev_correct": 49, "local_accuracy": 0.5, "jev_accuracy": 0.49, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -12.0, 15.0 ], "local_only_correct": 24, "jev_only_correct": 23, "both_correct": 26, "both_wrong": 27, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 38, "jev_correct": 48, "local_accuracy": 0.38, "jev_accuracy": 0.48, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -19.0, 0.0 ], "local_only_correct": 9, "jev_only_correct": 19, "both_correct": 29, "both_wrong": 43, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 80, "jev_correct": 79, "local_accuracy": 0.8, "jev_accuracy": 0.79, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 3, "both_correct": 76, "both_wrong": 17, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 7, "jev_correct": 6, "local_accuracy": 0.07, "jev_accuracy": 0.06, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 3, "both_correct": 3, "both_wrong": 90, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 58, "jev_correct": 64, "local_accuracy": 0.58, "jev_accuracy": 0.64, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -14.0, 2.0 ], "local_only_correct": 6, "jev_only_correct": 12, "both_correct": 52, "both_wrong": 30, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 41, "jev_correct": 37, "local_accuracy": 0.41, "jev_accuracy": 0.37, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -5.0, 13.0 ], "local_only_correct": 13, "jev_only_correct": 9, "both_correct": 28, "both_wrong": 50, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 3, "jev_correct": 16, "local_accuracy": 0.03, "jev_accuracy": 0.16, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -21.0, -6.0 ], "local_only_correct": 2, "jev_only_correct": 15, "both_correct": 1, "both_wrong": 82, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 19, "jev_correct": 34, "local_accuracy": 0.19, "jev_accuracy": 0.34, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -25.0, -5.0 ], "local_only_correct": 7, "jev_only_correct": 22, "both_correct": 12, "both_wrong": 59, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 76, "jev_correct": 83, "local_accuracy": 0.76, "jev_accuracy": 0.83, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -13.0, -2.0 ], "local_only_correct": 1, "jev_only_correct": 8, "both_correct": 75, "both_wrong": 16, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 18, "jev_correct": 21, "local_accuracy": 0.18, "jev_accuracy": 0.21, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -13.0, 7.0 ], "local_only_correct": 12, "jev_only_correct": 15, "both_correct": 6, "both_wrong": 67, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 44, "jev_correct": 51, "local_accuracy": 0.44, "jev_accuracy": 0.51, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -18.0, 4.0 ], "local_only_correct": 12, "jev_only_correct": 19, "both_correct": 32, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 55, "jev_correct": 65, "local_accuracy": 0.55, "jev_accuracy": 0.65, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 7, "jev_only_correct": 17, "both_correct": 48, "both_wrong": 28, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 49, "jev_correct": 39, "local_accuracy": 0.49, "jev_accuracy": 0.39, "difference_pp": 10.0, "paired_cluster_bootstrap95_pp": [ 1.0, 19.0 ], "local_only_correct": 16, "jev_only_correct": 6, "both_correct": 33, "both_wrong": 45, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 11, "jev_correct": 9, "local_accuracy": 0.11, "jev_accuracy": 0.09, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -5.0, 9.0 ], "local_only_correct": 8, "jev_only_correct": 6, "both_correct": 3, "both_wrong": 83, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 66, "jev_correct": 59, "local_accuracy": 0.66, "jev_accuracy": 0.59, "difference_pp": 7.000000000000001, "paired_cluster_bootstrap95_pp": [ -2.0, 15.0 ], "local_only_correct": 13, "jev_only_correct": 6, "both_correct": 53, "both_wrong": 28, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 80, "jev_correct": 91, "local_accuracy": 0.8, "jev_accuracy": 0.91, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -18.0, -5.0 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 80, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 90, "jev_correct": 88, "local_accuracy": 0.9, "jev_accuracy": 0.88, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -3.0, 8.0 ], "local_only_correct": 5, "jev_only_correct": 3, "both_correct": 85, "both_wrong": 7, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 49, "jev_correct": 48, "local_accuracy": 0.49, "jev_accuracy": 0.48, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -8.0, 10.024999999999864 ], "local_only_correct": 12, "jev_only_correct": 11, "both_correct": 37, "both_wrong": 40, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2165, "jev_correct": 2550, "local_accuracy": 0.7343962008141113, "jev_accuracy": 0.8649932157394844, "difference_pp": -13.059701492537313, "paired_cluster_bootstrap95_pp": [ -14.450474898236092, -11.668928086838534 ], "local_only_correct": 61, "jev_only_correct": 446, "both_correct": 2104, "both_wrong": 337, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 1898, "jev_correct": 2412, "local_accuracy": 0.6438263229308006, "jev_accuracy": 0.8181818181818182, "difference_pp": -17.435549525101766, "paired_cluster_bootstrap95_pp": [ -19.132462686567166, -15.739484396200814 ], "local_only_correct": 106, "jev_only_correct": 620, "both_correct": 1792, "both_wrong": 430, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 671, "jev_correct": 843, "local_accuracy": 0.5720375106564365, "jev_accuracy": 0.7186700767263428, "difference_pp": -14.663256606990622, "paired_cluster_bootstrap95_pp": [ -18.076832699922516, -11.194648291714687 ], "local_only_correct": 84, "jev_only_correct": 256, "both_correct": 587, "both_wrong": 246, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "gliner-decide-1b": { "fresh/news": { "n": 200, "local_correct": 142, "jev_correct": 178, "local_accuracy": 0.71, "jev_accuracy": 0.89, "difference_pp": -18.0, "paired_cluster_bootstrap95_pp": [ -23.5, -12.487500000000068 ], "local_only_correct": 3, "jev_only_correct": 39, "both_correct": 139, "both_wrong": 19, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 91, "jev_correct": 120, "local_accuracy": 0.455, "jev_accuracy": 0.6, "difference_pp": -14.499999999999998, "paired_cluster_bootstrap95_pp": [ -22.5, -6.5 ], "local_only_correct": 22, "jev_only_correct": 51, "both_correct": 69, "both_wrong": 58, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 173, "jev_correct": 196, "local_accuracy": 0.865, "jev_accuracy": 0.98, "difference_pp": -11.5, "paired_cluster_bootstrap95_pp": [ -16.5, -7.0 ], "local_only_correct": 2, "jev_only_correct": 25, "both_correct": 171, "both_wrong": 2, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 122, "jev_correct": 179, "local_accuracy": 0.61, "jev_accuracy": 0.895, "difference_pp": -28.499999999999996, "paired_cluster_bootstrap95_pp": [ -35.0, -22.0 ], "local_only_correct": 2, "jev_only_correct": 59, "both_correct": 120, "both_wrong": 19, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 7, "jev_correct": 11, "local_accuracy": 0.5833333333333334, "jev_accuracy": 0.9166666666666666, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -58.333333333333336, -8.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 7, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 2, "jev_correct": 10, "local_accuracy": 0.16666666666666666, "jev_accuracy": 0.8333333333333334, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -100.0, -25.0 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 1, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 10, "jev_correct": 12, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 10, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 7, "jev_correct": 12, "local_accuracy": 0.5833333333333334, "jev_accuracy": 1.0, "difference_pp": -41.66666666666667, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 7, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 74, "jev_correct": 91, "local_accuracy": 0.74, "jev_accuracy": 0.91, "difference_pp": -17.0, "paired_cluster_bootstrap95_pp": [ -26.0, -9.0 ], "local_only_correct": 2, "jev_only_correct": 19, "both_correct": 72, "both_wrong": 7, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 48, "jev_correct": 48, "local_accuracy": 0.48, "jev_accuracy": 0.48, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -12.0, 12.0 ], "local_only_correct": 18, "jev_only_correct": 18, "both_correct": 30, "both_wrong": 34, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 44, "jev_correct": 89, "local_accuracy": 0.44, "jev_accuracy": 0.89, "difference_pp": -45.0, "paired_cluster_bootstrap95_pp": [ -55.0, -34.0 ], "local_only_correct": 1, "jev_only_correct": 46, "both_correct": 43, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 47, "jev_correct": 48, "local_accuracy": 0.9791666666666666, "jev_accuracy": 1.0, "difference_pp": -2.083333333333333, "paired_cluster_bootstrap95_pp": [ -6.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 47, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/hard": { "n": 111, "local_correct": 39, "jev_correct": 81, "local_accuracy": 0.35135135135135137, "jev_accuracy": 0.7297297297297297, "difference_pp": -37.83783783783784, "paired_cluster_bootstrap95_pp": [ -47.747747747747745, -27.92792792792793 ], "local_only_correct": 2, "jev_only_correct": 44, "both_correct": 37, "both_wrong": 28, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 35, "jev_correct": 70, "local_accuracy": 0.4861111111111111, "jev_accuracy": 0.9722222222222222, "difference_pp": -48.61111111111111, "paired_cluster_bootstrap95_pp": [ -62.5, -33.333333333333336 ], "local_only_correct": 0, "jev_only_correct": 35, "both_correct": 35, "both_wrong": 2, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7368421052631579, "jev_accuracy": 0.9473684210526315, "difference_pp": -21.052631578947366, "paired_cluster_bootstrap95_pp": [ -36.8421052631579, -5.2631578947368425 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 6, "jev_correct": 16, "local_accuracy": 0.375, "jev_accuracy": 1.0, "difference_pp": -62.5, "paired_cluster_bootstrap95_pp": [ -87.5, -37.5 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 6, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 15, "jev_correct": 20, "local_accuracy": 0.6818181818181818, "jev_accuracy": 0.9090909090909091, "difference_pp": -22.727272727272727, "paired_cluster_bootstrap95_pp": [ -40.90909090909091, -8.977272727273348 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 15, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 12, "jev_correct": 18, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 12, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 16, "jev_correct": 19, "local_accuracy": 0.8421052631578947, "jev_accuracy": 1.0, "difference_pp": -15.789473684210526, "paired_cluster_bootstrap95_pp": [ -31.57894736842105, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 16, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.7647058823529411, "jev_accuracy": 0.9411764705882353, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 11, "jev_correct": 18, "local_accuracy": 0.6111111111111112, "jev_accuracy": 1.0, "difference_pp": -38.88888888888889, "paired_cluster_bootstrap95_pp": [ -61.111111111111114, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 11, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 18, "jev_correct": 21, "local_accuracy": 0.8181818181818182, "jev_accuracy": 0.9545454545454546, "difference_pp": -13.636363636363635, "paired_cluster_bootstrap95_pp": [ -27.272727272727273, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 18, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5, "jev_accuracy": 0.9444444444444444, "difference_pp": -44.44444444444444, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -22.22222222222222 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9444444444444444, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -5.555555555555555 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.75, "jev_accuracy": 0.9, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -35.0, 5.0 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 4, "jev_correct": 18, "local_accuracy": 0.19047619047619047, "jev_accuracy": 0.8571428571428571, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -85.71428571428571, -47.61904761904762 ], "local_only_correct": 0, "jev_only_correct": 14, "both_correct": 4, "both_wrong": 3, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.5625, "jev_accuracy": 1.0, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 19, "jev_correct": 23, "local_accuracy": 0.8260869565217391, "jev_accuracy": 1.0, "difference_pp": -17.391304347826086, "paired_cluster_bootstrap95_pp": [ -34.78260869565217, -4.3478260869565215 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 19, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 8, "jev_correct": 14, "local_accuracy": 0.5714285714285714, "jev_accuracy": 1.0, "difference_pp": -42.857142857142854, "paired_cluster_bootstrap95_pp": [ -71.42857142857143, -21.250000000000973 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 8, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 9, "jev_correct": 14, "local_accuracy": 0.5625, "jev_accuracy": 0.875, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 9, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 7, "jev_correct": 14, "local_accuracy": 0.4375, "jev_accuracy": 0.875, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -12.5 ], "local_only_correct": 1, "jev_only_correct": 8, "both_correct": 6, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 8, "jev_correct": 16, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 8, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 7, "jev_correct": 16, "local_accuracy": 0.4375, "jev_accuracy": 1.0, "difference_pp": -56.25, "paired_cluster_bootstrap95_pp": [ -81.25, -31.25 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 7, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7777777777777778, "jev_accuracy": 1.0, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 6, "jev_correct": 17, "local_accuracy": 0.35294117647058826, "jev_accuracy": 1.0, "difference_pp": -64.70588235294117, "paired_cluster_bootstrap95_pp": [ -88.23529411764706, -41.1764705882353 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 6, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 10, "jev_correct": 18, "local_accuracy": 0.5555555555555556, "jev_accuracy": 1.0, "difference_pp": -44.44444444444444, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -22.22222222222222 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 10, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5555555555555556, "jev_accuracy": 0.9444444444444444, "difference_pp": -38.88888888888889, "paired_cluster_bootstrap95_pp": [ -61.111111111111114, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4666666666666667, "jev_accuracy": 1.0, "difference_pp": -53.333333333333336, "paired_cluster_bootstrap95_pp": [ -80.0, -26.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 7, "jev_correct": 17, "local_accuracy": 0.3684210526315789, "jev_accuracy": 0.8947368421052632, "difference_pp": -52.63157894736842, "paired_cluster_bootstrap95_pp": [ -78.94736842105263, -26.31578947368421 ], "local_only_correct": 1, "jev_only_correct": 11, "both_correct": 6, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 6, "jev_correct": 17, "local_accuracy": 0.35294117647058826, "jev_accuracy": 1.0, "difference_pp": -64.70588235294117, "paired_cluster_bootstrap95_pp": [ -82.3529411764706, -41.1764705882353 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 6, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 12, "jev_correct": 19, "local_accuracy": 0.6, "jev_accuracy": 0.95, "difference_pp": -35.0, "paired_cluster_bootstrap95_pp": [ -60.0, -10.0 ], "local_only_correct": 1, "jev_only_correct": 8, "both_correct": 11, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 18, "jev_correct": 20, "local_accuracy": 0.9, "jev_accuracy": 1.0, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 18, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 0.8, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -25.0, 15.0 ], "local_only_correct": 2, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 7, "jev_correct": 20, "local_accuracy": 0.3181818181818182, "jev_accuracy": 0.9090909090909091, "difference_pp": -59.09090909090909, "paired_cluster_bootstrap95_pp": [ -77.38636363636363, -36.36363636363637 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 7, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 472, "jev_correct": 629, "local_accuracy": 0.5244444444444445, "jev_accuracy": 0.6988888888888889, "difference_pp": -17.444444444444446, "paired_cluster_bootstrap95_pp": [ -22.238270459254142, -13.059514176107415 ], "local_only_correct": 84, "jev_only_correct": 241, "both_correct": 388, "both_wrong": 187, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 242, "jev_correct": 465, "local_accuracy": 0.42160278745644597, "jev_accuracy": 0.8101045296167247, "difference_pp": -38.850174216027874, "paired_cluster_bootstrap95_pp": [ -43.20557491289198, -34.146341463414636 ], "local_only_correct": 22, "jev_only_correct": 245, "both_correct": 220, "both_wrong": 87, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 135, "jev_correct": 490, "local_accuracy": 0.19285714285714287, "jev_accuracy": 0.7, "difference_pp": -50.71428571428571, "paired_cluster_bootstrap95_pp": [ -55.19240446994032, -46.23020319891495 ], "local_only_correct": 35, "jev_only_correct": 390, "both_correct": 100, "both_wrong": 175, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 13, "jev_correct": 97, "local_accuracy": 0.0325, "jev_accuracy": 0.2425, "difference_pp": -21.0, "paired_cluster_bootstrap95_pp": [ -25.5, -16.5 ], "local_only_correct": 12, "jev_only_correct": 96, "both_correct": 1, "both_wrong": 291, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 26, "jev_correct": 28, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -17.857142857142858, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 26, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 21, "jev_correct": 28, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, -10.714285714285714 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 21, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 22, "jev_correct": 28, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -39.285714285714285, -7.142857142857143 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 22, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 31, "jev_correct": 40, "local_accuracy": 0.775, "jev_accuracy": 1.0, "difference_pp": -22.5, "paired_cluster_bootstrap95_pp": [ -37.5, -10.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 31, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 32, "jev_correct": 40, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -32.5, -7.5 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 32, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 22, "jev_correct": 39, "local_accuracy": 0.55, "jev_accuracy": 0.975, "difference_pp": -42.5, "paired_cluster_bootstrap95_pp": [ -57.5, -27.5 ], "local_only_correct": 0, "jev_only_correct": 17, "both_correct": 22, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 23, "jev_correct": 39, "local_accuracy": 0.575, "jev_accuracy": 0.975, "difference_pp": -40.0, "paired_cluster_bootstrap95_pp": [ -55.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 16, "both_correct": 23, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 196, "jev_correct": 237, "local_accuracy": 0.49, "jev_accuracy": 0.5925, "difference_pp": -10.25, "paired_cluster_bootstrap95_pp": [ -15.5, -4.75 ], "local_only_correct": 42, "jev_only_correct": 83, "both_correct": 154, "both_wrong": 121, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 232, "jev_correct": 277, "local_accuracy": 0.7733333333333333, "jev_accuracy": 0.9233333333333333, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -19.666666666666668, -10.333333333333334 ], "local_only_correct": 5, "jev_only_correct": 50, "both_correct": 227, "both_wrong": 18, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 116, "jev_correct": 377, "local_accuracy": 0.29, "jev_accuracy": 0.9425, "difference_pp": -65.25, "paired_cluster_bootstrap95_pp": [ -70.0, -60.5 ], "local_only_correct": 5, "jev_only_correct": 266, "both_correct": 111, "both_wrong": 18, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 285, "jev_correct": 346, "local_accuracy": 0.7125, "jev_accuracy": 0.865, "difference_pp": -15.25, "paired_cluster_bootstrap95_pp": [ -19.25, -11.5 ], "local_only_correct": 4, "jev_only_correct": 65, "both_correct": 281, "both_wrong": 50, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 233, "jev_correct": 360, "local_accuracy": 0.5825, "jev_accuracy": 0.9, "difference_pp": -31.75, "paired_cluster_bootstrap95_pp": [ -36.25, -27.0 ], "local_only_correct": 2, "jev_only_correct": 129, "both_correct": 231, "both_wrong": 38, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 155, "jev_correct": 225, "local_accuracy": 0.5166666666666667, "jev_accuracy": 0.75, "difference_pp": -23.333333333333332, "paired_cluster_bootstrap95_pp": [ -29.333333333333332, -17.333333333333332 ], "local_only_correct": 14, "jev_only_correct": 84, "both_correct": 141, "both_wrong": 61, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 339, "jev_correct": 393, "local_accuracy": 0.8475, "jev_accuracy": 0.9825, "difference_pp": -13.5, "paired_cluster_bootstrap95_pp": [ -17.25, -9.75 ], "local_only_correct": 6, "jev_only_correct": 60, "both_correct": 333, "both_wrong": 1, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 206, "jev_correct": 265, "local_accuracy": 0.515, "jev_accuracy": 0.6625, "difference_pp": -14.75, "paired_cluster_bootstrap95_pp": [ -23.00625, -6.5 ], "local_only_correct": 125, "jev_only_correct": 184, "both_correct": 81, "both_wrong": 10, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 77, "jev_correct": 75, "local_accuracy": 0.77, "jev_accuracy": 0.75, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -8.0, 12.0 ], "local_only_correct": 14, "jev_only_correct": 12, "both_correct": 63, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 58, "jev_correct": 56, "local_accuracy": 0.58, "jev_accuracy": 0.56, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -9.0, 13.0 ], "local_only_correct": 16, "jev_only_correct": 14, "both_correct": 42, "both_wrong": 28, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 59, "jev_correct": 49, "local_accuracy": 0.59, "jev_accuracy": 0.49, "difference_pp": 10.0, "paired_cluster_bootstrap95_pp": [ -6.0, 26.0 ], "local_only_correct": 36, "jev_only_correct": 26, "both_correct": 23, "both_wrong": 15, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 45, "jev_correct": 48, "local_accuracy": 0.45, "jev_accuracy": 0.48, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -10.0, 3.0 ], "local_only_correct": 4, "jev_only_correct": 7, "both_correct": 41, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 78, "jev_correct": 79, "local_accuracy": 0.78, "jev_accuracy": 0.79, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -9.0, 6.0 ], "local_only_correct": 7, "jev_only_correct": 8, "both_correct": 71, "both_wrong": 14, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 17, "jev_correct": 6, "local_accuracy": 0.17, "jev_accuracy": 0.06, "difference_pp": 11.0, "paired_cluster_bootstrap95_pp": [ 4.0, 18.0 ], "local_only_correct": 13, "jev_only_correct": 2, "both_correct": 4, "both_wrong": 81, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 63, "jev_correct": 64, "local_accuracy": 0.63, "jev_accuracy": 0.64, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -10.0, 7.024999999999864 ], "local_only_correct": 9, "jev_only_correct": 10, "both_correct": 54, "both_wrong": 27, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 46, "jev_correct": 37, "local_accuracy": 0.46, "jev_accuracy": 0.37, "difference_pp": 9.0, "paired_cluster_bootstrap95_pp": [ -2.0, 21.0 ], "local_only_correct": 24, "jev_only_correct": 15, "both_correct": 22, "both_wrong": 39, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 5, "jev_correct": 16, "local_accuracy": 0.05, "jev_accuracy": 0.16, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -20.0, -3.0 ], "local_only_correct": 5, "jev_only_correct": 16, "both_correct": 0, "both_wrong": 79, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 6, "jev_correct": 34, "local_accuracy": 0.06, "jev_accuracy": 0.34, "difference_pp": -28.000000000000004, "paired_cluster_bootstrap95_pp": [ -38.0, -18.0 ], "local_only_correct": 3, "jev_only_correct": 31, "both_correct": 3, "both_wrong": 63, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 82, "jev_correct": 83, "local_accuracy": 0.82, "jev_accuracy": 0.83, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -6.0, 4.0 ], "local_only_correct": 3, "jev_only_correct": 4, "both_correct": 79, "both_wrong": 14, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 0, "jev_correct": 21, "local_accuracy": 0.0, "jev_accuracy": 0.21, "difference_pp": -21.0, "paired_cluster_bootstrap95_pp": [ -29.025, -13.0 ], "local_only_correct": 0, "jev_only_correct": 21, "both_correct": 0, "both_wrong": 79, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 33, "jev_correct": 51, "local_accuracy": 0.33, "jev_accuracy": 0.51, "difference_pp": -18.0, "paired_cluster_bootstrap95_pp": [ -29.0, -6.0 ], "local_only_correct": 10, "jev_only_correct": 28, "both_correct": 23, "both_wrong": 39, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 69, "jev_correct": 65, "local_accuracy": 0.69, "jev_accuracy": 0.65, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -2.0, 10.0 ], "local_only_correct": 7, "jev_only_correct": 3, "both_correct": 62, "both_wrong": 28, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 42, "jev_correct": 39, "local_accuracy": 0.42, "jev_accuracy": 0.39, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -5.0, 11.0 ], "local_only_correct": 9, "jev_only_correct": 6, "both_correct": 33, "both_wrong": 52, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 14, "jev_correct": 9, "local_accuracy": 0.14, "jev_accuracy": 0.09, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -4.0, 14.0 ], "local_only_correct": 13, "jev_only_correct": 8, "both_correct": 1, "both_wrong": 78, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 61, "jev_correct": 59, "local_accuracy": 0.61, "jev_accuracy": 0.59, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -7.0, 11.0 ], "local_only_correct": 11, "jev_only_correct": 9, "both_correct": 50, "both_wrong": 30, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 73, "jev_correct": 91, "local_accuracy": 0.73, "jev_accuracy": 0.91, "difference_pp": -18.0, "paired_cluster_bootstrap95_pp": [ -26.0, -11.0 ], "local_only_correct": 0, "jev_only_correct": 18, "both_correct": 73, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 76, "jev_correct": 88, "local_accuracy": 0.76, "jev_accuracy": 0.88, "difference_pp": -12.0, "paired_cluster_bootstrap95_pp": [ -20.0, -5.0 ], "local_only_correct": 2, "jev_only_correct": 14, "both_correct": 74, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 49, "jev_correct": 48, "local_accuracy": 0.49, "jev_accuracy": 0.48, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -11.0, 13.0 ], "local_only_correct": 19, "jev_only_correct": 18, "both_correct": 30, "both_wrong": 33, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 1729, "jev_correct": 2550, "local_accuracy": 0.5864993215739485, "jev_accuracy": 0.8649932157394844, "difference_pp": -27.849389416553592, "paired_cluster_bootstrap95_pp": [ -29.681139755766623, -25.94979647218453 ], "local_only_correct": 77, "jev_only_correct": 898, "both_correct": 1652, "both_wrong": 321, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 729, "jev_correct": 2412, "local_accuracy": 0.2472862957937585, "jev_accuracy": 0.8181818181818182, "difference_pp": -57.08955223880597, "paired_cluster_bootstrap95_pp": [ -58.95522388059702, -55.25780189959294 ], "local_only_correct": 57, "jev_only_correct": 1740, "both_correct": 672, "both_wrong": 479, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 484, "jev_correct": 843, "local_accuracy": 0.41261722080136404, "jev_accuracy": 0.7186700767263428, "difference_pp": -30.605285592497864, "paired_cluster_bootstrap95_pp": [ -35.63098352838068, -25.774041909011235 ], "local_only_correct": 122, "jev_only_correct": 481, "both_correct": 362, "both_wrong": 208, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "gliner-decide-multi": { "fresh/news": { "n": 200, "local_correct": 149, "jev_correct": 178, "local_accuracy": 0.745, "jev_accuracy": 0.89, "difference_pp": -14.499999999999998, "paired_cluster_bootstrap95_pp": [ -20.0, -9.5 ], "local_only_correct": 3, "jev_only_correct": 32, "both_correct": 146, "both_wrong": 19, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 78, "jev_correct": 120, "local_accuracy": 0.39, "jev_accuracy": 0.6, "difference_pp": -21.0, "paired_cluster_bootstrap95_pp": [ -28.5, -13.5 ], "local_only_correct": 11, "jev_only_correct": 53, "both_correct": 67, "both_wrong": 69, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 125, "jev_correct": 196, "local_accuracy": 0.625, "jev_accuracy": 0.98, "difference_pp": -35.5, "paired_cluster_bootstrap95_pp": [ -43.0, -29.0 ], "local_only_correct": 1, "jev_only_correct": 72, "both_correct": 124, "both_wrong": 3, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 129, "jev_correct": 179, "local_accuracy": 0.645, "jev_accuracy": 0.895, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -31.5, -18.5 ], "local_only_correct": 4, "jev_only_correct": 54, "both_correct": 125, "both_wrong": 17, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 5, "jev_correct": 11, "local_accuracy": 0.4166666666666667, "jev_accuracy": 0.9166666666666666, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 5, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 3, "jev_correct": 10, "local_accuracy": 0.25, "jev_accuracy": 0.8333333333333334, "difference_pp": -58.333333333333336, "paired_cluster_bootstrap95_pp": [ -83.33333333333333, -33.333333333333336 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 3, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 3, "jev_correct": 12, "local_accuracy": 0.25, "jev_accuracy": 1.0, "difference_pp": -75.0, "paired_cluster_bootstrap95_pp": [ -100.0, -50.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 3, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 6, "jev_correct": 12, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 6, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 78, "jev_correct": 91, "local_accuracy": 0.78, "jev_accuracy": 0.91, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -22.0, -5.0 ], "local_only_correct": 4, "jev_only_correct": 17, "both_correct": 74, "both_wrong": 5, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 49, "jev_correct": 48, "local_accuracy": 0.49, "jev_accuracy": 0.48, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -10.0, 12.0 ], "local_only_correct": 16, "jev_only_correct": 15, "both_correct": 33, "both_wrong": 36, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 61, "jev_correct": 89, "local_accuracy": 0.61, "jev_accuracy": 0.89, "difference_pp": -28.000000000000004, "paired_cluster_bootstrap95_pp": [ -37.0, -20.0 ], "local_only_correct": 0, "jev_only_correct": 28, "both_correct": 61, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 47, "jev_correct": 48, "local_accuracy": 0.9791666666666666, "jev_accuracy": 1.0, "difference_pp": -2.083333333333333, "paired_cluster_bootstrap95_pp": [ -6.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 47, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/hard": { "n": 111, "local_correct": 42, "jev_correct": 81, "local_accuracy": 0.3783783783783784, "jev_accuracy": 0.7297297297297297, "difference_pp": -35.13513513513514, "paired_cluster_bootstrap95_pp": [ -45.945945945945944, -23.423423423423422 ], "local_only_correct": 9, "jev_only_correct": 48, "both_correct": 33, "both_wrong": 21, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 42, "jev_correct": 70, "local_accuracy": 0.5833333333333334, "jev_accuracy": 0.9722222222222222, "difference_pp": -38.88888888888889, "paired_cluster_bootstrap95_pp": [ -54.166666666666664, -23.61111111111111 ], "local_only_correct": 1, "jev_only_correct": 29, "both_correct": 41, "both_wrong": 1, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.6842105263157895, "jev_accuracy": 0.9473684210526315, "difference_pp": -26.31578947368421, "paired_cluster_bootstrap95_pp": [ -47.36842105263158, -10.526315789473685 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 13, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -43.75, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 12, "jev_correct": 20, "local_accuracy": 0.5454545454545454, "jev_accuracy": 0.9090909090909091, "difference_pp": -36.36363636363637, "paired_cluster_bootstrap95_pp": [ -54.54545454545455, -18.181818181818183 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 12, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 4, "jev_correct": 18, "local_accuracy": 0.2222222222222222, "jev_accuracy": 1.0, "difference_pp": -77.77777777777779, "paired_cluster_bootstrap95_pp": [ -94.44444444444444, -55.55555555555556 ], "local_only_correct": 0, "jev_only_correct": 14, "both_correct": 4, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 8, "jev_correct": 16, "local_accuracy": 0.47058823529411764, "jev_accuracy": 0.9411764705882353, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 8, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4666666666666667, "jev_accuracy": 1.0, "difference_pp": -53.333333333333336, "paired_cluster_bootstrap95_pp": [ -80.0, -26.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4375, "jev_accuracy": 0.9375, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -18.75 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 6, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 2, "jev_correct": 18, "local_accuracy": 0.1111111111111111, "jev_accuracy": 1.0, "difference_pp": -88.88888888888889, "paired_cluster_bootstrap95_pp": [ -100.0, -72.22222222222223 ], "local_only_correct": 0, "jev_only_correct": 16, "both_correct": 2, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 5, "jev_correct": 21, "local_accuracy": 0.22727272727272727, "jev_accuracy": 0.9545454545454546, "difference_pp": -72.72727272727273, "paired_cluster_bootstrap95_pp": [ -90.9090909090909, -54.54545454545455 ], "local_only_correct": 0, "jev_only_correct": 16, "both_correct": 5, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 9, "jev_correct": 19, "local_accuracy": 0.47368421052631576, "jev_accuracy": 1.0, "difference_pp": -52.63157894736842, "paired_cluster_bootstrap95_pp": [ -73.6842105263158, -31.57894736842105 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 9, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -60.0, -13.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 9, "jev_correct": 18, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -72.22222222222223, -27.77777777777778 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 9, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.4444444444444444, "jev_accuracy": 0.9444444444444444, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -72.22222222222223, -27.77777777777778 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 4, "jev_correct": 15, "local_accuracy": 0.25, "jev_accuracy": 0.9375, "difference_pp": -68.75, "paired_cluster_bootstrap95_pp": [ -87.5, -43.75 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 4, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5555555555555556, "jev_accuracy": 0.9444444444444444, "difference_pp": -38.88888888888889, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 8, "jev_correct": 18, "local_accuracy": 0.4, "jev_accuracy": 0.9, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -70.0, -30.0 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 8, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 5, "jev_correct": 18, "local_accuracy": 0.23809523809523808, "jev_accuracy": 0.8571428571428571, "difference_pp": -61.904761904761905, "paired_cluster_bootstrap95_pp": [ -85.71428571428571, -33.333333333333336 ], "local_only_correct": 1, "jev_only_correct": 14, "both_correct": 4, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 11, "jev_correct": 16, "local_accuracy": 0.6875, "jev_accuracy": 1.0, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 11, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 19, "jev_correct": 23, "local_accuracy": 0.8260869565217391, "jev_accuracy": 1.0, "difference_pp": -17.391304347826086, "paired_cluster_bootstrap95_pp": [ -34.78260869565217, -4.3478260869565215 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 19, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 4, "jev_correct": 14, "local_accuracy": 0.2857142857142857, "jev_accuracy": 1.0, "difference_pp": -71.42857142857143, "paired_cluster_bootstrap95_pp": [ -92.85714285714286, -42.857142857142854 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 4, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 3, "jev_correct": 14, "local_accuracy": 0.1875, "jev_accuracy": 0.875, "difference_pp": -68.75, "paired_cluster_bootstrap95_pp": [ -87.5, -43.75 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 3, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 8, "jev_correct": 14, "local_accuracy": 0.5, "jev_accuracy": 0.875, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -68.75, -6.25 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 7, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.5625, "jev_accuracy": 1.0, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 5, "jev_correct": 16, "local_accuracy": 0.3125, "jev_accuracy": 1.0, "difference_pp": -68.75, "paired_cluster_bootstrap95_pp": [ -87.5, -43.75 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 5, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 8, "jev_correct": 18, "local_accuracy": 0.4444444444444444, "jev_accuracy": 1.0, "difference_pp": -55.55555555555556, "paired_cluster_bootstrap95_pp": [ -77.77777777777777, -33.333333333333336 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 8, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 5, "jev_correct": 17, "local_accuracy": 0.29411764705882354, "jev_accuracy": 1.0, "difference_pp": -70.58823529411765, "paired_cluster_bootstrap95_pp": [ -88.23529411764706, -47.05882352941177 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 5, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 9, "jev_correct": 18, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -72.22222222222223, -27.77777777777778 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 9, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.4444444444444444, "jev_accuracy": 0.9444444444444444, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -72.22222222222223, -27.77777777777778 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 5, "jev_correct": 15, "local_accuracy": 0.3125, "jev_accuracy": 0.9375, "difference_pp": -62.5, "paired_cluster_bootstrap95_pp": [ -87.5, -37.5 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 5, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4666666666666667, "jev_accuracy": 1.0, "difference_pp": -53.333333333333336, "paired_cluster_bootstrap95_pp": [ -80.0, -26.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 5, "jev_correct": 17, "local_accuracy": 0.2631578947368421, "jev_accuracy": 0.8947368421052632, "difference_pp": -63.1578947368421, "paired_cluster_bootstrap95_pp": [ -89.47368421052632, -36.8421052631579 ], "local_only_correct": 1, "jev_only_correct": 13, "both_correct": 4, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 9, "jev_correct": 19, "local_accuracy": 0.45, "jev_accuracy": 0.95, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 1, "jev_only_correct": 11, "both_correct": 8, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 16, "jev_correct": 20, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, -5.0 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 16, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.65, "jev_accuracy": 0.8, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -45.0, 15.0 ], "local_only_correct": 3, "jev_only_correct": 6, "both_correct": 10, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 7, "jev_correct": 20, "local_accuracy": 0.3181818181818182, "jev_accuracy": 0.9090909090909091, "difference_pp": -59.09090909090909, "paired_cluster_bootstrap95_pp": [ -77.38636363636363, -36.36363636363637 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 7, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 393, "jev_correct": 629, "local_accuracy": 0.43666666666666665, "jev_accuracy": 0.6988888888888889, "difference_pp": -26.222222222222225, "paired_cluster_bootstrap95_pp": [ -31.99655783727236, -21.225461088851254 ], "local_only_correct": 74, "jev_only_correct": 310, "both_correct": 319, "both_wrong": 197, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 242, "jev_correct": 465, "local_accuracy": 0.42160278745644597, "jev_accuracy": 0.8101045296167247, "difference_pp": -38.850174216027874, "paired_cluster_bootstrap95_pp": [ -43.20557491289198, -34.146341463414636 ], "local_only_correct": 19, "jev_only_correct": 242, "both_correct": 223, "both_wrong": 90, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 116, "jev_correct": 490, "local_accuracy": 0.1657142857142857, "jev_accuracy": 0.7, "difference_pp": -53.42857142857142, "paired_cluster_bootstrap95_pp": [ -57.71509054325956, -49.08055714386028 ], "local_only_correct": 31, "jev_only_correct": 405, "both_correct": 85, "both_wrong": 179, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 7, "jev_correct": 97, "local_accuracy": 0.0175, "jev_accuracy": 0.2425, "difference_pp": -22.5, "paired_cluster_bootstrap95_pp": [ -26.5, -18.5 ], "local_only_correct": 2, "jev_only_correct": 92, "both_correct": 5, "both_wrong": 301, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 22, "jev_correct": 28, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -35.714285714285715, -7.142857142857143 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 22, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 21, "jev_correct": 28, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, -10.714285714285714 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 21, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 21, "jev_correct": 28, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, -10.714285714285714 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 21, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 18, "jev_correct": 28, "local_accuracy": 0.6428571428571429, "jev_accuracy": 1.0, "difference_pp": -35.714285714285715, "paired_cluster_bootstrap95_pp": [ -53.57142857142857, -17.857142857142858 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 18, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 23, "jev_correct": 40, "local_accuracy": 0.575, "jev_accuracy": 1.0, "difference_pp": -42.5, "paired_cluster_bootstrap95_pp": [ -57.5, -27.5 ], "local_only_correct": 0, "jev_only_correct": 17, "both_correct": 23, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 27, "jev_correct": 40, "local_accuracy": 0.675, "jev_accuracy": 1.0, "difference_pp": -32.5, "paired_cluster_bootstrap95_pp": [ -47.5, -17.5 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 27, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 24, "jev_correct": 39, "local_accuracy": 0.6, "jev_accuracy": 0.975, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -52.5, -22.5 ], "local_only_correct": 0, "jev_only_correct": 15, "both_correct": 24, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 27, "jev_correct": 39, "local_accuracy": 0.675, "jev_accuracy": 0.975, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -42.5625, -17.43750000000034 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 27, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 171, "jev_correct": 237, "local_accuracy": 0.4275, "jev_accuracy": 0.5925, "difference_pp": -16.5, "paired_cluster_bootstrap95_pp": [ -22.25, -10.5 ], "local_only_correct": 40, "jev_only_correct": 106, "both_correct": 131, "both_wrong": 123, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 210, "jev_correct": 277, "local_accuracy": 0.7, "jev_accuracy": 0.9233333333333333, "difference_pp": -22.333333333333332, "paired_cluster_bootstrap95_pp": [ -27.666666666666668, -17.333333333333332 ], "local_only_correct": 5, "jev_only_correct": 72, "both_correct": 205, "both_wrong": 18, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 315, "jev_correct": 377, "local_accuracy": 0.7875, "jev_accuracy": 0.9425, "difference_pp": -15.5, "paired_cluster_bootstrap95_pp": [ -20.0, -11.25 ], "local_only_correct": 13, "jev_only_correct": 75, "both_correct": 302, "both_wrong": 10, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 295, "jev_correct": 346, "local_accuracy": 0.7375, "jev_accuracy": 0.865, "difference_pp": -12.75, "paired_cluster_bootstrap95_pp": [ -16.5, -9.25 ], "local_only_correct": 7, "jev_only_correct": 58, "both_correct": 288, "both_wrong": 47, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 256, "jev_correct": 360, "local_accuracy": 0.64, "jev_accuracy": 0.9, "difference_pp": -26.0, "paired_cluster_bootstrap95_pp": [ -31.0, -21.0 ], "local_only_correct": 11, "jev_only_correct": 115, "both_correct": 245, "both_wrong": 29, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 172, "jev_correct": 225, "local_accuracy": 0.5733333333333334, "jev_accuracy": 0.75, "difference_pp": -17.666666666666668, "paired_cluster_bootstrap95_pp": [ -23.666666666666668, -11.991666666666712 ], "local_only_correct": 18, "jev_only_correct": 71, "both_correct": 154, "both_wrong": 57, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 211, "jev_correct": 393, "local_accuracy": 0.5275, "jev_accuracy": 0.9825, "difference_pp": -45.5, "paired_cluster_bootstrap95_pp": [ -50.75, -40.5 ], "local_only_correct": 3, "jev_only_correct": 185, "both_correct": 208, "both_wrong": 4, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 229, "jev_correct": 265, "local_accuracy": 0.5725, "jev_accuracy": 0.6625, "difference_pp": -9.0, "paired_cluster_bootstrap95_pp": [ -14.0, -4.493750000000034 ], "local_only_correct": 30, "jev_only_correct": 66, "both_correct": 199, "both_wrong": 105, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 68, "jev_correct": 75, "local_accuracy": 0.68, "jev_accuracy": 0.75, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -17.0, 3.0 ], "local_only_correct": 11, "jev_only_correct": 18, "both_correct": 57, "both_wrong": 14, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 54, "jev_correct": 56, "local_accuracy": 0.54, "jev_accuracy": 0.56, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -13.0, 9.0 ], "local_only_correct": 14, "jev_only_correct": 16, "both_correct": 40, "both_wrong": 30, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 50, "jev_correct": 49, "local_accuracy": 0.5, "jev_accuracy": 0.49, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -12.0, 15.0 ], "local_only_correct": 23, "jev_only_correct": 22, "both_correct": 27, "both_wrong": 28, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 39, "jev_correct": 48, "local_accuracy": 0.39, "jev_accuracy": 0.48, "difference_pp": -9.0, "paired_cluster_bootstrap95_pp": [ -19.0, 0.0 ], "local_only_correct": 8, "jev_only_correct": 17, "both_correct": 31, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 72, "jev_correct": 79, "local_accuracy": 0.72, "jev_accuracy": 0.79, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -14.0, 0.0 ], "local_only_correct": 4, "jev_only_correct": 11, "both_correct": 68, "both_wrong": 17, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 12, "jev_correct": 6, "local_accuracy": 0.12, "jev_accuracy": 0.06, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ -1.0, 13.0 ], "local_only_correct": 9, "jev_only_correct": 3, "both_correct": 3, "both_wrong": 85, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 56, "jev_correct": 64, "local_accuracy": 0.56, "jev_accuracy": 0.64, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -19.0, 2.0 ], "local_only_correct": 9, "jev_only_correct": 17, "both_correct": 47, "both_wrong": 27, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 40, "jev_correct": 37, "local_accuracy": 0.4, "jev_accuracy": 0.37, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -8.024999999999999, 15.0 ], "local_only_correct": 20, "jev_only_correct": 17, "both_correct": 20, "both_wrong": 43, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.14, "jev_accuracy": 0.16, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -13.0, 8.024999999999864 ], "local_only_correct": 14, "jev_only_correct": 16, "both_correct": 0, "both_wrong": 70, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 3, "jev_correct": 34, "local_accuracy": 0.03, "jev_accuracy": 0.34, "difference_pp": -31.0, "paired_cluster_bootstrap95_pp": [ -41.0, -21.0 ], "local_only_correct": 2, "jev_only_correct": 33, "both_correct": 1, "both_wrong": 64, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 78, "jev_correct": 83, "local_accuracy": 0.78, "jev_accuracy": 0.83, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -13.0, 3.0 ], "local_only_correct": 5, "jev_only_correct": 10, "both_correct": 73, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 1, "jev_correct": 21, "local_accuracy": 0.01, "jev_accuracy": 0.21, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -29.0, -12.0 ], "local_only_correct": 1, "jev_only_correct": 21, "both_correct": 0, "both_wrong": 78, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 23, "jev_correct": 51, "local_accuracy": 0.23, "jev_accuracy": 0.51, "difference_pp": -28.000000000000004, "paired_cluster_bootstrap95_pp": [ -40.0, -16.0 ], "local_only_correct": 8, "jev_only_correct": 36, "both_correct": 15, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 63, "jev_correct": 65, "local_accuracy": 0.63, "jev_accuracy": 0.65, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -9.0, 5.0 ], "local_only_correct": 6, "jev_only_correct": 8, "both_correct": 57, "both_wrong": 29, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 46, "jev_correct": 39, "local_accuracy": 0.46, "jev_accuracy": 0.39, "difference_pp": 7.000000000000001, "paired_cluster_bootstrap95_pp": [ -1.0, 15.0 ], "local_only_correct": 13, "jev_only_correct": 6, "both_correct": 33, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 12, "jev_correct": 9, "local_accuracy": 0.12, "jev_accuracy": 0.09, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -4.0, 10.0 ], "local_only_correct": 8, "jev_only_correct": 5, "both_correct": 4, "both_wrong": 83, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 52, "jev_correct": 59, "local_accuracy": 0.52, "jev_accuracy": 0.59, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -15.0, 1.0 ], "local_only_correct": 6, "jev_only_correct": 13, "both_correct": 46, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 79, "jev_correct": 91, "local_accuracy": 0.79, "jev_accuracy": 0.91, "difference_pp": -12.0, "paired_cluster_bootstrap95_pp": [ -21.0, -3.0 ], "local_only_correct": 6, "jev_only_correct": 18, "both_correct": 73, "both_wrong": 3, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 34, "jev_correct": 88, "local_accuracy": 0.34, "jev_accuracy": 0.88, "difference_pp": -54.0, "paired_cluster_bootstrap95_pp": [ -64.0, -44.0 ], "local_only_correct": 0, "jev_only_correct": 54, "both_correct": 34, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 50, "jev_correct": 48, "local_accuracy": 0.5, "jev_accuracy": 0.48, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -9.0, 13.0 ], "local_only_correct": 17, "jev_only_correct": 15, "both_correct": 33, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 1625, "jev_correct": 2550, "local_accuracy": 0.5512211668928086, "jev_accuracy": 0.8649932157394844, "difference_pp": -31.37720488466757, "paired_cluster_bootstrap95_pp": [ -33.310719131614654, -29.44369063772049 ], "local_only_correct": 75, "jev_only_correct": 1000, "both_correct": 1550, "both_wrong": 323, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 1455, "jev_correct": 2412, "local_accuracy": 0.4935549525101764, "jev_accuracy": 0.8181818181818182, "difference_pp": -32.46268656716418, "paired_cluster_bootstrap95_pp": [ -34.4640434192673, -30.42740841248304 ], "local_only_correct": 121, "jev_only_correct": 1078, "both_correct": 1334, "both_wrong": 415, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 780, "jev_correct": 843, "local_accuracy": 0.6649616368286445, "jev_accuracy": 0.7186700767263428, "difference_pp": -5.3708439897698215, "paired_cluster_bootstrap95_pp": [ -9.752671374439126, -1.3911158350713952 ], "local_only_correct": 180, "jev_only_correct": 243, "both_correct": 600, "both_wrong": 150, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "gliner-decide": { "fresh/news": { "n": 200, "local_correct": 140, "jev_correct": 178, "local_accuracy": 0.7, "jev_accuracy": 0.89, "difference_pp": -19.0, "paired_cluster_bootstrap95_pp": [ -25.0, -13.0 ], "local_only_correct": 4, "jev_only_correct": 42, "both_correct": 136, "both_wrong": 18, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 106, "jev_correct": 120, "local_accuracy": 0.53, "jev_accuracy": 0.6, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -13.5, 0.0 ], "local_only_correct": 19, "jev_only_correct": 33, "both_correct": 87, "both_wrong": 61, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 143, "jev_correct": 196, "local_accuracy": 0.715, "jev_accuracy": 0.98, "difference_pp": -26.5, "paired_cluster_bootstrap95_pp": [ -33.0, -20.5 ], "local_only_correct": 0, "jev_only_correct": 53, "both_correct": 143, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 127, "jev_correct": 179, "local_accuracy": 0.635, "jev_accuracy": 0.895, "difference_pp": -26.0, "paired_cluster_bootstrap95_pp": [ -32.0, -20.0 ], "local_only_correct": 0, "jev_only_correct": 52, "both_correct": 127, "both_wrong": 21, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 8, "jev_correct": 11, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9166666666666666, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 8, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 4, "jev_correct": 10, "local_accuracy": 0.3333333333333333, "jev_accuracy": 0.8333333333333334, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 4, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 10, "jev_correct": 12, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 10, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 11, "jev_correct": 12, "local_accuracy": 0.9166666666666666, "jev_accuracy": 1.0, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 11, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 87, "jev_correct": 91, "local_accuracy": 0.87, "jev_accuracy": 0.91, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -10.0, 2.0 ], "local_only_correct": 3, "jev_only_correct": 7, "both_correct": 84, "both_wrong": 6, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 56, "jev_correct": 48, "local_accuracy": 0.56, "jev_accuracy": 0.48, "difference_pp": 8.0, "paired_cluster_bootstrap95_pp": [ -2.0, 18.0 ], "local_only_correct": 18, "jev_only_correct": 10, "both_correct": 38, "both_wrong": 34, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 75, "jev_correct": 89, "local_accuracy": 0.75, "jev_accuracy": 0.89, "difference_pp": -14.000000000000002, "paired_cluster_bootstrap95_pp": [ -22.0, -6.0 ], "local_only_correct": 2, "jev_only_correct": 16, "both_correct": 73, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 47, "jev_correct": 48, "local_accuracy": 0.9791666666666666, "jev_accuracy": 1.0, "difference_pp": -2.083333333333333, "paired_cluster_bootstrap95_pp": [ -6.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 47, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/hard": { "n": 111, "local_correct": 36, "jev_correct": 81, "local_accuracy": 0.32432432432432434, "jev_accuracy": 0.7297297297297297, "difference_pp": -40.54054054054054, "paired_cluster_bootstrap95_pp": [ -50.450450450450454, -29.72972972972973 ], "local_only_correct": 3, "jev_only_correct": 48, "both_correct": 33, "both_wrong": 27, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 48, "jev_correct": 70, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9722222222222222, "difference_pp": -30.555555555555557, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -18.055555555555557 ], "local_only_correct": 2, "jev_only_correct": 24, "both_correct": 46, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.7894736842105263, "jev_accuracy": 0.9473684210526315, "difference_pp": -15.789473684210526, "paired_cluster_bootstrap95_pp": [ -31.57894736842105, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 10, "jev_correct": 16, "local_accuracy": 0.625, "jev_accuracy": 1.0, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 10, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 12, "jev_correct": 20, "local_accuracy": 0.5454545454545454, "jev_accuracy": 0.9090909090909091, "difference_pp": -36.36363636363637, "paired_cluster_bootstrap95_pp": [ -63.63636363636363, -9.090909090909092 ], "local_only_correct": 2, "jev_only_correct": 10, "both_correct": 10, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.7058823529411765, "jev_accuracy": 0.9411764705882353, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8125, "jev_accuracy": 0.9375, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7777777777777778, "jev_accuracy": 1.0, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -39.027777777777764, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 18, "jev_correct": 21, "local_accuracy": 0.8181818181818182, "jev_accuracy": 0.9545454545454546, "difference_pp": -13.636363636363635, "paired_cluster_bootstrap95_pp": [ -27.272727272727273, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 18, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7777777777777778, "jev_accuracy": 1.0, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7777777777777778, "jev_accuracy": 0.9444444444444444, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -38.888888888888886, 5.555555555555555 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7222222222222222, "jev_accuracy": 0.9444444444444444, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.75, "jev_accuracy": 0.9, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -35.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 6, "jev_correct": 18, "local_accuracy": 0.2857142857142857, "jev_accuracy": 0.8571428571428571, "difference_pp": -57.14285714285714, "paired_cluster_bootstrap95_pp": [ -80.95238095238095, -28.571428571428573 ], "local_only_correct": 2, "jev_only_correct": 14, "both_correct": 4, "both_wrong": 1, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 11, "jev_correct": 16, "local_accuracy": 0.6875, "jev_accuracy": 1.0, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -6.25 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 11, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 20, "jev_correct": 23, "local_accuracy": 0.8695652173913043, "jev_accuracy": 1.0, "difference_pp": -13.043478260869565, "paired_cluster_bootstrap95_pp": [ -26.08695652173913, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 20, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.75, "jev_accuracy": 0.875, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -43.75, 12.5 ], "local_only_correct": 2, "jev_only_correct": 4, "both_correct": 10, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.75, "jev_accuracy": 0.875, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -37.5, 12.5 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 10, "jev_correct": 16, "local_accuracy": 0.625, "jev_accuracy": 1.0, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 10, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 5, "jev_correct": 17, "local_accuracy": 0.29411764705882354, "jev_accuracy": 1.0, "difference_pp": -70.58823529411765, "paired_cluster_bootstrap95_pp": [ -88.23529411764706, -47.05882352941177 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 5, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9444444444444444, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -5.416666666667425 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 6, "jev_correct": 15, "local_accuracy": 0.375, "jev_accuracy": 0.9375, "difference_pp": -56.25, "paired_cluster_bootstrap95_pp": [ -81.25, -31.25 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 6, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 9, "jev_correct": 15, "local_accuracy": 0.6, "jev_accuracy": 1.0, "difference_pp": -40.0, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -13.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 9, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.42105263157894735, "jev_accuracy": 0.8947368421052632, "difference_pp": -47.368421052631575, "paired_cluster_bootstrap95_pp": [ -73.6842105263158, -21.05263157894737 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 7, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 6, "jev_correct": 17, "local_accuracy": 0.35294117647058826, "jev_accuracy": 1.0, "difference_pp": -64.70588235294117, "paired_cluster_bootstrap95_pp": [ -82.3529411764706, -41.1764705882353 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 6, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7777777777777778, "jev_accuracy": 1.0, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 14, "jev_correct": 19, "local_accuracy": 0.7, "jev_accuracy": 0.95, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -45.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 14, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 18, "jev_correct": 20, "local_accuracy": 0.9, "jev_accuracy": 1.0, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 18, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 10, "jev_correct": 16, "local_accuracy": 0.5, "jev_accuracy": 0.8, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -60.0, 5.0 ], "local_only_correct": 4, "jev_only_correct": 10, "both_correct": 6, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 443, "jev_correct": 629, "local_accuracy": 0.4922222222222222, "jev_accuracy": 0.6988888888888889, "difference_pp": -20.666666666666668, "paired_cluster_bootstrap95_pp": [ -25.314901316838313, -16.512316887783307 ], "local_only_correct": 72, "jev_only_correct": 258, "both_correct": 371, "both_wrong": 199, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 237, "jev_correct": 465, "local_accuracy": 0.41289198606271776, "jev_accuracy": 0.8101045296167247, "difference_pp": -39.721254355400696, "paired_cluster_bootstrap95_pp": [ -44.07665505226481, -35.01742160278746 ], "local_only_correct": 19, "jev_only_correct": 247, "both_correct": 218, "both_wrong": 90, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 162, "jev_correct": 490, "local_accuracy": 0.23142857142857143, "jev_accuracy": 0.7, "difference_pp": -46.85714285714286, "paired_cluster_bootstrap95_pp": [ -51.28589901477832, -42.53504937672009 ], "local_only_correct": 34, "jev_only_correct": 362, "both_correct": 128, "both_wrong": 176, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 12, "jev_correct": 97, "local_accuracy": 0.03, "jev_accuracy": 0.2425, "difference_pp": -21.25, "paired_cluster_bootstrap95_pp": [ -25.5, -16.75 ], "local_only_correct": 10, "jev_only_correct": 95, "both_correct": 2, "both_wrong": 293, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 26, "jev_correct": 28, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -17.857142857142858, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 26, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 24, "jev_correct": 28, "local_accuracy": 0.8571428571428571, "jev_accuracy": 1.0, "difference_pp": -14.285714285714285, "paired_cluster_bootstrap95_pp": [ -28.571428571428573, -3.5714285714285716 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 24, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 22, "jev_correct": 28, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -35.714285714285715, -7.142857142857143 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 22, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 22, "jev_correct": 28, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -35.714285714285715, -7.142857142857143 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 22, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 33, "jev_correct": 40, "local_accuracy": 0.825, "jev_accuracy": 1.0, "difference_pp": -17.5, "paired_cluster_bootstrap95_pp": [ -30.0, -7.5 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 33, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 34, "jev_correct": 40, "local_accuracy": 0.85, "jev_accuracy": 1.0, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -27.5, -5.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 34, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 30, "jev_correct": 39, "local_accuracy": 0.75, "jev_accuracy": 0.975, "difference_pp": -22.5, "paired_cluster_bootstrap95_pp": [ -37.5, -10.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 30, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 26, "jev_correct": 39, "local_accuracy": 0.65, "jev_accuracy": 0.975, "difference_pp": -32.5, "paired_cluster_bootstrap95_pp": [ -47.5, -20.0 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 26, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 222, "jev_correct": 237, "local_accuracy": 0.555, "jev_accuracy": 0.5925, "difference_pp": -3.75, "paired_cluster_bootstrap95_pp": [ -8.5, 1.25 ], "local_only_correct": 43, "jev_only_correct": 58, "both_correct": 179, "both_wrong": 120, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 246, "jev_correct": 277, "local_accuracy": 0.82, "jev_accuracy": 0.9233333333333333, "difference_pp": -10.333333333333334, "paired_cluster_bootstrap95_pp": [ -14.333333333333334, -6.666666666666667 ], "local_only_correct": 4, "jev_only_correct": 35, "both_correct": 242, "both_wrong": 19, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 366, "jev_correct": 377, "local_accuracy": 0.915, "jev_accuracy": 0.9425, "difference_pp": -2.75, "paired_cluster_bootstrap95_pp": [ -5.5, 0.0 ], "local_only_correct": 11, "jev_only_correct": 22, "both_correct": 355, "both_wrong": 12, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 284, "jev_correct": 346, "local_accuracy": 0.71, "jev_accuracy": 0.865, "difference_pp": -15.5, "paired_cluster_bootstrap95_pp": [ -19.5, -11.5 ], "local_only_correct": 5, "jev_only_correct": 67, "both_correct": 279, "both_wrong": 49, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 258, "jev_correct": 360, "local_accuracy": 0.645, "jev_accuracy": 0.9, "difference_pp": -25.5, "paired_cluster_bootstrap95_pp": [ -30.0, -21.0 ], "local_only_correct": 4, "jev_only_correct": 106, "both_correct": 254, "both_wrong": 36, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 187, "jev_correct": 225, "local_accuracy": 0.6233333333333333, "jev_accuracy": 0.75, "difference_pp": -12.666666666666668, "paired_cluster_bootstrap95_pp": [ -18.0, -7.333333333333333 ], "local_only_correct": 16, "jev_only_correct": 54, "both_correct": 171, "both_wrong": 59, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 275, "jev_correct": 393, "local_accuracy": 0.6875, "jev_accuracy": 0.9825, "difference_pp": -29.5, "paired_cluster_bootstrap95_pp": [ -34.25, -25.0 ], "local_only_correct": 1, "jev_only_correct": 119, "both_correct": 274, "both_wrong": 6, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 262, "jev_correct": 265, "local_accuracy": 0.655, "jev_accuracy": 0.6625, "difference_pp": -0.75, "paired_cluster_bootstrap95_pp": [ -5.5, 3.75 ], "local_only_correct": 40, "jev_only_correct": 43, "both_correct": 222, "both_wrong": 95, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 81, "jev_correct": 75, "local_accuracy": 0.81, "jev_accuracy": 0.75, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ -4.0, 16.0 ], "local_only_correct": 15, "jev_only_correct": 9, "both_correct": 66, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 64, "jev_correct": 56, "local_accuracy": 0.64, "jev_accuracy": 0.56, "difference_pp": 8.0, "paired_cluster_bootstrap95_pp": [ -2.0, 18.0 ], "local_only_correct": 18, "jev_only_correct": 10, "both_correct": 46, "both_wrong": 26, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 57, "jev_correct": 49, "local_accuracy": 0.57, "jev_accuracy": 0.49, "difference_pp": 8.0, "paired_cluster_bootstrap95_pp": [ -5.0, 21.0 ], "local_only_correct": 27, "jev_only_correct": 19, "both_correct": 30, "both_wrong": 24, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 42, "jev_correct": 48, "local_accuracy": 0.42, "jev_accuracy": 0.48, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -13.0, 0.0 ], "local_only_correct": 3, "jev_only_correct": 9, "both_correct": 39, "both_wrong": 49, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 80, "jev_correct": 79, "local_accuracy": 0.8, "jev_accuracy": 0.79, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -6.0, 8.0 ], "local_only_correct": 7, "jev_only_correct": 6, "both_correct": 73, "both_wrong": 14, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 17, "jev_correct": 6, "local_accuracy": 0.17, "jev_accuracy": 0.06, "difference_pp": 11.0, "paired_cluster_bootstrap95_pp": [ 3.0, 19.0 ], "local_only_correct": 14, "jev_only_correct": 3, "both_correct": 3, "both_wrong": 80, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 58, "jev_correct": 64, "local_accuracy": 0.58, "jev_accuracy": 0.64, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -17.0, 5.0 ], "local_only_correct": 12, "jev_only_correct": 18, "both_correct": 46, "both_wrong": 24, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 42, "jev_correct": 37, "local_accuracy": 0.42, "jev_accuracy": 0.37, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -6.0, 16.0 ], "local_only_correct": 19, "jev_only_correct": 14, "both_correct": 23, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 18, "jev_correct": 16, "local_accuracy": 0.18, "jev_accuracy": 0.16, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -9.0, 12.0 ], "local_only_correct": 16, "jev_only_correct": 14, "both_correct": 2, "both_wrong": 68, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 5, "jev_correct": 34, "local_accuracy": 0.05, "jev_accuracy": 0.34, "difference_pp": -28.999999999999996, "paired_cluster_bootstrap95_pp": [ -39.0, -19.0 ], "local_only_correct": 2, "jev_only_correct": 31, "both_correct": 3, "both_wrong": 64, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 84, "jev_correct": 83, "local_accuracy": 0.84, "jev_accuracy": 0.83, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 3, "both_correct": 80, "both_wrong": 13, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 5, "jev_correct": 21, "local_accuracy": 0.05, "jev_accuracy": 0.21, "difference_pp": -16.0, "paired_cluster_bootstrap95_pp": [ -24.0, -8.0 ], "local_only_correct": 2, "jev_only_correct": 18, "both_correct": 3, "both_wrong": 77, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 40, "jev_correct": 51, "local_accuracy": 0.4, "jev_accuracy": 0.51, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -21.0, -0.9750000000001364 ], "local_only_correct": 9, "jev_only_correct": 20, "both_correct": 31, "both_wrong": 40, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 69, "jev_correct": 65, "local_accuracy": 0.69, "jev_accuracy": 0.65, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -2.0, 11.0 ], "local_only_correct": 8, "jev_only_correct": 4, "both_correct": 61, "both_wrong": 27, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 48, "jev_correct": 39, "local_accuracy": 0.48, "jev_accuracy": 0.39, "difference_pp": 9.0, "paired_cluster_bootstrap95_pp": [ 0.0, 18.0 ], "local_only_correct": 15, "jev_only_correct": 6, "both_correct": 33, "both_wrong": 46, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 14, "jev_correct": 9, "local_accuracy": 0.14, "jev_accuracy": 0.09, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -3.0, 13.0 ], "local_only_correct": 11, "jev_only_correct": 6, "both_correct": 3, "both_wrong": 80, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 63, "jev_correct": 59, "local_accuracy": 0.63, "jev_accuracy": 0.59, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -4.0, 12.0 ], "local_only_correct": 11, "jev_only_correct": 7, "both_correct": 52, "both_wrong": 30, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 71, "jev_correct": 91, "local_accuracy": 0.71, "jev_accuracy": 0.91, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -28.0, -12.0 ], "local_only_correct": 0, "jev_only_correct": 20, "both_correct": 71, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 77, "jev_correct": 88, "local_accuracy": 0.77, "jev_accuracy": 0.88, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -20.0, -3.0 ], "local_only_correct": 4, "jev_only_correct": 15, "both_correct": 73, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 52, "jev_correct": 48, "local_accuracy": 0.52, "jev_accuracy": 0.48, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -5.0, 14.0 ], "local_only_correct": 14, "jev_only_correct": 10, "both_correct": 38, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2067, "jev_correct": 2550, "local_accuracy": 0.7011533242876526, "jev_accuracy": 0.8649932157394844, "difference_pp": -16.383989145183175, "paired_cluster_bootstrap95_pp": [ -17.91044776119403, -14.823609226594302 ], "local_only_correct": 56, "jev_only_correct": 539, "both_correct": 2011, "both_wrong": 342, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 1557, "jev_correct": 2412, "local_accuracy": 0.5281546811397557, "jev_accuracy": 0.8181818181818182, "difference_pp": -29.00271370420624, "paired_cluster_bootstrap95_pp": [ -30.834464043419267, -27.137042062415198 ], "local_only_correct": 85, "jev_only_correct": 940, "both_correct": 1472, "both_wrong": 451, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 703, "jev_correct": 843, "local_accuracy": 0.5993179880647911, "jev_accuracy": 0.7186700767263428, "difference_pp": -11.935208866155158, "paired_cluster_bootstrap95_pp": [ -16.031397912788368, -8.112339629490029 ], "local_only_correct": 136, "jev_only_correct": 276, "both_correct": 567, "both_wrong": 194, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "imajev-2b": { "fresh/news": { "n": 200, "local_correct": 168, "jev_correct": 178, "local_accuracy": 0.84, "jev_accuracy": 0.89, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -9.0, -1.0 ], "local_only_correct": 3, "jev_only_correct": 13, "both_correct": 165, "both_wrong": 19, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 121, "jev_correct": 120, "local_accuracy": 0.605, "jev_accuracy": 0.6, "difference_pp": 0.5, "paired_cluster_bootstrap95_pp": [ -4.5, 5.5 ], "local_only_correct": 12, "jev_only_correct": 11, "both_correct": 109, "both_wrong": 68, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 136, "jev_correct": 196, "local_accuracy": 0.68, "jev_accuracy": 0.98, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -36.5, -24.0 ], "local_only_correct": 0, "jev_only_correct": 60, "both_correct": 136, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 144, "jev_correct": 179, "local_accuracy": 0.72, "jev_accuracy": 0.895, "difference_pp": -17.5, "paired_cluster_bootstrap95_pp": [ -23.5, -12.0 ], "local_only_correct": 3, "jev_only_correct": 38, "both_correct": 141, "both_wrong": 18, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 9, "jev_correct": 11, "local_accuracy": 0.75, "jev_accuracy": 0.9166666666666666, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 9, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 8, "jev_correct": 10, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.8333333333333334, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 8, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 83, "jev_correct": 91, "local_accuracy": 0.83, "jev_accuracy": 0.91, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -14.0, -2.0 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 82, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 45, "jev_correct": 48, "local_accuracy": 0.45, "jev_accuracy": 0.48, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -10.0, 4.0 ], "local_only_correct": 5, "jev_only_correct": 8, "both_correct": 40, "both_wrong": 47, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 77, "jev_correct": 89, "local_accuracy": 0.77, "jev_accuracy": 0.89, "difference_pp": -12.0, "paired_cluster_bootstrap95_pp": [ -20.0, -5.0 ], "local_only_correct": 2, "jev_only_correct": 14, "both_correct": 75, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 47, "jev_correct": 48, "local_accuracy": 0.9791666666666666, "jev_accuracy": 1.0, "difference_pp": -2.083333333333333, "paired_cluster_bootstrap95_pp": [ -6.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 47, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/hard": { "n": 111, "local_correct": 62, "jev_correct": 81, "local_accuracy": 0.5585585585585585, "jev_accuracy": 0.7297297297297297, "difference_pp": -17.117117117117118, "paired_cluster_bootstrap95_pp": [ -27.027027027027028, -7.207207207207207 ], "local_only_correct": 7, "jev_only_correct": 26, "both_correct": 55, "both_wrong": 23, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 65, "jev_correct": 70, "local_accuracy": 0.9027777777777778, "jev_accuracy": 0.9722222222222222, "difference_pp": -6.944444444444445, "paired_cluster_bootstrap95_pp": [ -15.277777777777779, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 64, "both_wrong": 1, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8421052631578947, "jev_accuracy": 0.9473684210526315, "difference_pp": -10.526315789473683, "paired_cluster_bootstrap95_pp": [ -26.31578947368421, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 18, "jev_correct": 20, "local_accuracy": 0.8181818181818182, "jev_accuracy": 0.9090909090909091, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -27.272727272727273, 9.090909090909092 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 17, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.7222222222222222, "jev_accuracy": 1.0, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -10.97222222222298 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.7058823529411765, "jev_accuracy": 0.9411764705882353, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8125, "jev_accuracy": 0.9375, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 12, "jev_correct": 18, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 12, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 19, "jev_correct": 21, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9545454545454546, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -22.727272727272727, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9444444444444444, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -22.22222222222222, 11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 9, "jev_correct": 15, "local_accuracy": 0.5625, "jev_accuracy": 0.9375, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 9, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5555555555555556, "jev_accuracy": 0.9444444444444444, "difference_pp": -38.88888888888889, "paired_cluster_bootstrap95_pp": [ -61.111111111111114, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8, "jev_accuracy": 0.9, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 6, "jev_correct": 18, "local_accuracy": 0.2857142857142857, "jev_accuracy": 0.8571428571428571, "difference_pp": -57.14285714285714, "paired_cluster_bootstrap95_pp": [ -80.95238095238095, -28.571428571428573 ], "local_only_correct": 1, "jev_only_correct": 13, "both_correct": 5, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.8125, "jev_accuracy": 1.0, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 13, "jev_correct": 14, "local_accuracy": 0.8125, "jev_accuracy": 0.875, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -31.25, 12.5 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 9, "jev_correct": 14, "local_accuracy": 0.5625, "jev_accuracy": 0.875, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 9, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 11, "jev_correct": 16, "local_accuracy": 0.6875, "jev_accuracy": 1.0, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 11, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 5, "jev_correct": 17, "local_accuracy": 0.29411764705882354, "jev_accuracy": 1.0, "difference_pp": -70.58823529411765, "paired_cluster_bootstrap95_pp": [ -88.23529411764706, -47.05882352941177 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 5, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -38.888888888888886, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 7, "jev_correct": 17, "local_accuracy": 0.3888888888888889, "jev_accuracy": 0.9444444444444444, "difference_pp": -55.55555555555556, "paired_cluster_bootstrap95_pp": [ -77.77777777777777, -33.333333333333336 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 7, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.6875, "jev_accuracy": 0.9375, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5333333333333333, "jev_accuracy": 1.0, "difference_pp": -46.666666666666664, "paired_cluster_bootstrap95_pp": [ -73.33333333333333, -20.0 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.631578947368421, "jev_accuracy": 0.8947368421052632, "difference_pp": -26.31578947368421, "paired_cluster_bootstrap95_pp": [ -52.63157894736842, -5.2631578947368425 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 15, "jev_correct": 19, "local_accuracy": 0.75, "jev_accuracy": 0.95, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, 0.12499999999931788 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 14, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.45, "jev_accuracy": 0.8, "difference_pp": -35.0, "paired_cluster_bootstrap95_pp": [ -55.0, -15.0 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 4, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 22, "jev_correct": 20, "local_accuracy": 1.0, "jev_accuracy": 0.9090909090909091, "difference_pp": 9.090909090909092, "paired_cluster_bootstrap95_pp": [ 0.0, 22.727272727272727 ], "local_only_correct": 2, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 498, "jev_correct": 629, "local_accuracy": 0.5533333333333333, "jev_accuracy": 0.6988888888888889, "difference_pp": -14.555555555555555, "paired_cluster_bootstrap95_pp": [ -19.488795666548143, -10.123176092301254 ], "local_only_correct": 58, "jev_only_correct": 189, "both_correct": 440, "both_wrong": 213, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 345, "jev_correct": 465, "local_accuracy": 0.6010452961672473, "jev_accuracy": 0.8101045296167247, "difference_pp": -20.905923344947734, "paired_cluster_bootstrap95_pp": [ -25.087108013937282, -16.72473867595819 ], "local_only_correct": 24, "jev_only_correct": 144, "both_correct": 321, "both_wrong": 85, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 222, "jev_correct": 490, "local_accuracy": 0.3171428571428571, "jev_accuracy": 0.7, "difference_pp": -38.285714285714285, "paired_cluster_bootstrap95_pp": [ -42.566737496392285, -34.03291720146272 ], "local_only_correct": 29, "jev_only_correct": 297, "both_correct": 193, "both_wrong": 181, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 37, "jev_correct": 97, "local_accuracy": 0.0925, "jev_accuracy": 0.2425, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -19.25, -10.75 ], "local_only_correct": 16, "jev_only_correct": 76, "both_correct": 21, "both_wrong": 287, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 24, "jev_correct": 28, "local_accuracy": 0.8571428571428571, "jev_accuracy": 1.0, "difference_pp": -14.285714285714285, "paired_cluster_bootstrap95_pp": [ -28.571428571428573, -3.5714285714285716 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 24, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 36, "jev_correct": 40, "local_accuracy": 0.9, "jev_accuracy": 1.0, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -20.0, -2.5 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 36, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 36, "jev_correct": 40, "local_accuracy": 0.9, "jev_accuracy": 1.0, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -20.0, -2.5 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 36, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 32, "jev_correct": 39, "local_accuracy": 0.8, "jev_accuracy": 0.975, "difference_pp": -17.5, "paired_cluster_bootstrap95_pp": [ -30.0, -7.5 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 32, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 30, "jev_correct": 39, "local_accuracy": 0.75, "jev_accuracy": 0.975, "difference_pp": -22.5, "paired_cluster_bootstrap95_pp": [ -35.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 30, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 239, "jev_correct": 237, "local_accuracy": 0.5975, "jev_accuracy": 0.5925, "difference_pp": 0.5, "paired_cluster_bootstrap95_pp": [ -3.25, 4.25 ], "local_only_correct": 29, "jev_only_correct": 27, "both_correct": 210, "both_wrong": 134, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 278, "jev_correct": 277, "local_accuracy": 0.9266666666666666, "jev_accuracy": 0.9233333333333333, "difference_pp": 0.33333333333333337, "paired_cluster_bootstrap95_pp": [ -2.3333333333333335, 3.3333333333333335 ], "local_only_correct": 10, "jev_only_correct": 9, "both_correct": 268, "both_wrong": 13, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 273, "jev_correct": 377, "local_accuracy": 0.6825, "jev_accuracy": 0.9425, "difference_pp": -26.0, "paired_cluster_bootstrap95_pp": [ -30.25625, -21.5 ], "local_only_correct": 3, "jev_only_correct": 107, "both_correct": 270, "both_wrong": 20, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 326, "jev_correct": 346, "local_accuracy": 0.815, "jev_accuracy": 0.865, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -7.75, -2.5 ], "local_only_correct": 3, "jev_only_correct": 23, "both_correct": 323, "both_wrong": 51, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 296, "jev_correct": 360, "local_accuracy": 0.74, "jev_accuracy": 0.9, "difference_pp": -16.0, "paired_cluster_bootstrap95_pp": [ -20.25, -12.25 ], "local_only_correct": 7, "jev_only_correct": 71, "both_correct": 289, "both_wrong": 33, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 228, "jev_correct": 225, "local_accuracy": 0.76, "jev_accuracy": 0.75, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -2.3333333333333335, 4.333333333333333 ], "local_only_correct": 14, "jev_only_correct": 11, "both_correct": 214, "both_wrong": 61, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 277, "jev_correct": 393, "local_accuracy": 0.6925, "jev_accuracy": 0.9825, "difference_pp": -28.999999999999996, "paired_cluster_bootstrap95_pp": [ -33.75, -24.25 ], "local_only_correct": 2, "jev_only_correct": 118, "both_correct": 275, "both_wrong": 5, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 305, "jev_correct": 265, "local_accuracy": 0.7625, "jev_accuracy": 0.6625, "difference_pp": 10.0, "paired_cluster_bootstrap95_pp": [ 5.5, 14.5 ], "local_only_correct": 68, "jev_only_correct": 28, "both_correct": 237, "both_wrong": 67, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 68, "jev_correct": 75, "local_accuracy": 0.68, "jev_accuracy": 0.75, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 4, "jev_only_correct": 11, "both_correct": 64, "both_wrong": 21, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 59, "jev_correct": 56, "local_accuracy": 0.59, "jev_accuracy": 0.56, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -4.0, 10.0 ], "local_only_correct": 8, "jev_only_correct": 5, "both_correct": 51, "both_wrong": 36, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 55, "jev_correct": 49, "local_accuracy": 0.55, "jev_accuracy": 0.49, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ -4.0, 16.0 ], "local_only_correct": 16, "jev_only_correct": 10, "both_correct": 39, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 43, "jev_correct": 48, "local_accuracy": 0.43, "jev_accuracy": 0.48, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -13.0, 3.0 ], "local_only_correct": 6, "jev_only_correct": 11, "both_correct": 37, "both_wrong": 46, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 75, "jev_correct": 79, "local_accuracy": 0.75, "jev_accuracy": 0.79, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -10.0, 1.0 ], "local_only_correct": 2, "jev_only_correct": 6, "both_correct": 73, "both_wrong": 19, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 8, "jev_correct": 6, "local_accuracy": 0.08, "jev_accuracy": 0.06, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -3.0, 7.0 ], "local_only_correct": 4, "jev_only_correct": 2, "both_correct": 4, "both_wrong": 90, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 57, "jev_correct": 64, "local_accuracy": 0.57, "jev_accuracy": 0.64, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -16.0, 1.0 ], "local_only_correct": 6, "jev_only_correct": 13, "both_correct": 51, "both_wrong": 30, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 43, "jev_correct": 37, "local_accuracy": 0.43, "jev_accuracy": 0.37, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ -1.0, 12.0 ], "local_only_correct": 9, "jev_only_correct": 3, "both_correct": 34, "both_wrong": 54, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 10, "jev_correct": 16, "local_accuracy": 0.1, "jev_accuracy": 0.16, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -12.0, 0.0 ], "local_only_correct": 3, "jev_only_correct": 9, "both_correct": 7, "both_wrong": 81, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 13, "jev_correct": 34, "local_accuracy": 0.13, "jev_accuracy": 0.34, "difference_pp": -21.0, "paired_cluster_bootstrap95_pp": [ -30.0, -11.0 ], "local_only_correct": 4, "jev_only_correct": 25, "both_correct": 9, "both_wrong": 62, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 84, "jev_correct": 83, "local_accuracy": 0.84, "jev_accuracy": 0.83, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 9.0 ], "local_only_correct": 9, "jev_only_correct": 8, "both_correct": 75, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 19, "jev_correct": 21, "local_accuracy": 0.19, "jev_accuracy": 0.21, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -12.0, 7.0 ], "local_only_correct": 11, "jev_only_correct": 13, "both_correct": 8, "both_wrong": 68, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 48, "jev_correct": 51, "local_accuracy": 0.48, "jev_accuracy": 0.51, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -11.0, 5.0 ], "local_only_correct": 8, "jev_only_correct": 11, "both_correct": 40, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 59, "jev_correct": 65, "local_accuracy": 0.59, "jev_accuracy": 0.65, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -14.0, 2.0 ], "local_only_correct": 6, "jev_only_correct": 12, "both_correct": 53, "both_wrong": 29, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 40, "jev_correct": 39, "local_accuracy": 0.4, "jev_accuracy": 0.39, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 7.0 ], "local_only_correct": 5, "jev_only_correct": 4, "both_correct": 35, "both_wrong": 56, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 6, "jev_correct": 9, "local_accuracy": 0.06, "jev_accuracy": 0.09, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -8.0, 1.0 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 5, "both_wrong": 90, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 63, "jev_correct": 59, "local_accuracy": 0.63, "jev_accuracy": 0.59, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -4.0, 12.0 ], "local_only_correct": 10, "jev_only_correct": 6, "both_correct": 53, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 87, "jev_correct": 91, "local_accuracy": 0.87, "jev_accuracy": 0.91, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -10.0, 1.0 ], "local_only_correct": 2, "jev_only_correct": 6, "both_correct": 85, "both_wrong": 7, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 83, "jev_correct": 88, "local_accuracy": 0.83, "jev_accuracy": 0.88, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -11.0, 1.0 ], "local_only_correct": 3, "jev_only_correct": 8, "both_correct": 80, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 46, "jev_correct": 48, "local_accuracy": 0.46, "jev_accuracy": 0.48, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -11.0, 7.0 ], "local_only_correct": 10, "jev_only_correct": 12, "both_correct": 36, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2439, "jev_correct": 2550, "local_accuracy": 0.8273405698778833, "jev_accuracy": 0.8649932157394844, "difference_pp": -3.7652645861601086, "paired_cluster_bootstrap95_pp": [ -4.952510176390773, -2.5780189959294435 ], "local_only_correct": 97, "jev_only_correct": 208, "both_correct": 2342, "both_wrong": 301, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2156, "jev_correct": 2412, "local_accuracy": 0.7313432835820896, "jev_accuracy": 0.8181818181818182, "difference_pp": -8.683853459972863, "paired_cluster_bootstrap95_pp": [ -10.074626865671641, -7.259158751696065 ], "local_only_correct": 104, "jev_only_correct": 360, "both_correct": 2052, "both_wrong": 432, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 770, "jev_correct": 843, "local_accuracy": 0.6564364876385337, "jev_accuracy": 0.7186700767263428, "difference_pp": -6.223358908780904, "paired_cluster_bootstrap95_pp": [ -10.753718114711623, -2.048722549972702 ], "local_only_correct": 177, "jev_only_correct": 250, "both_correct": 593, "both_wrong": 153, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "imajev-4b": { "fresh/news": { "n": 200, "local_correct": 168, "jev_correct": 178, "local_accuracy": 0.84, "jev_accuracy": 0.89, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -8.5, -2.0 ], "local_only_correct": 1, "jev_only_correct": 11, "both_correct": 167, "both_wrong": 21, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 80, "jev_correct": 120, "local_accuracy": 0.4, "jev_accuracy": 0.6, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -26.5, -13.5 ], "local_only_correct": 6, "jev_only_correct": 46, "both_correct": 74, "both_wrong": 74, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 169, "jev_correct": 196, "local_accuracy": 0.845, "jev_accuracy": 0.98, "difference_pp": -13.5, "paired_cluster_bootstrap95_pp": [ -18.0, -9.0 ], "local_only_correct": 0, "jev_only_correct": 27, "both_correct": 169, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 164, "jev_correct": 179, "local_accuracy": 0.82, "jev_accuracy": 0.895, "difference_pp": -7.5, "paired_cluster_bootstrap95_pp": [ -11.5, -3.5 ], "local_only_correct": 1, "jev_only_correct": 16, "both_correct": 163, "both_wrong": 20, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 10, "jev_correct": 11, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9166666666666666, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 10, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 9, "jev_correct": 10, "local_accuracy": 0.75, "jev_accuracy": 0.8333333333333334, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 9, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 86, "jev_correct": 91, "local_accuracy": 0.86, "jev_accuracy": 0.91, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -10.0, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 85, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 36, "jev_correct": 48, "local_accuracy": 0.36, "jev_accuracy": 0.48, "difference_pp": -12.0, "paired_cluster_bootstrap95_pp": [ -21.0, -3.0 ], "local_only_correct": 5, "jev_only_correct": 17, "both_correct": 31, "both_wrong": 47, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 88, "jev_correct": 89, "local_accuracy": 0.88, "jev_accuracy": 0.89, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -6.0, 4.0 ], "local_only_correct": 3, "jev_only_correct": 4, "both_correct": 85, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 80, "jev_correct": 81, "local_accuracy": 0.7207207207207207, "jev_accuracy": 0.7297297297297297, "difference_pp": -0.9009009009009009, "paired_cluster_bootstrap95_pp": [ -7.207207207207207, 5.405405405405405 ], "local_only_correct": 5, "jev_only_correct": 6, "both_correct": 75, "both_wrong": 25, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 70, "jev_correct": 70, "local_accuracy": 0.9722222222222222, "jev_accuracy": 0.9722222222222222, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -4.166666666666667, 4.166666666666667 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 69, "both_wrong": 1, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.9473684210526315, "jev_accuracy": 0.9473684210526315, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9090909090909091, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.7647058823529411, "jev_accuracy": 0.9411764705882353, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -41.1764705882353, 5.882352941176471 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8125, "jev_accuracy": 0.9375, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7777777777777778, "jev_accuracy": 1.0, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -38.888888888888886, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 20, "jev_correct": 21, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9545454545454546, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 20, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9444444444444444, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -22.22222222222222, 11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5, "jev_accuracy": 0.9375, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7777777777777778, "jev_accuracy": 0.9444444444444444, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.85, "jev_accuracy": 0.9, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -20.0, 10.0 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 9, "jev_correct": 18, "local_accuracy": 0.42857142857142855, "jev_accuracy": 0.8571428571428571, "difference_pp": -42.857142857142854, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -19.047619047619047 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 8, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 14, "jev_correct": 14, "local_accuracy": 0.875, "jev_accuracy": 0.875, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -18.75, 18.75 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 14, "jev_correct": 14, "local_accuracy": 0.875, "jev_accuracy": 0.875, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -18.75, 18.75 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6111111111111112, "jev_accuracy": 0.9444444444444444, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.6875, "jev_accuracy": 0.9375, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.631578947368421, "jev_accuracy": 0.8947368421052632, "difference_pp": -26.31578947368421, "paired_cluster_bootstrap95_pp": [ -52.63157894736842, 5.2631578947368425 ], "local_only_correct": 2, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 14, "jev_correct": 19, "local_accuracy": 0.7, "jev_accuracy": 0.95, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -45.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 14, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 0.8, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -20.0, 10.0 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 3, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 21, "jev_correct": 20, "local_accuracy": 0.9545454545454546, "jev_accuracy": 0.9090909090909091, "difference_pp": 4.545454545454546, "paired_cluster_bootstrap95_pp": [ 0.0, 13.636363636363637 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 553, "jev_correct": 629, "local_accuracy": 0.6144444444444445, "jev_accuracy": 0.6988888888888889, "difference_pp": -8.444444444444445, "paired_cluster_bootstrap95_pp": [ -12.838065983720714, -4.63189958165658 ], "local_only_correct": 47, "jev_only_correct": 123, "both_correct": 506, "both_wrong": 224, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 353, "jev_correct": 465, "local_accuracy": 0.6149825783972126, "jev_accuracy": 0.8101045296167247, "difference_pp": -19.51219512195122, "paired_cluster_bootstrap95_pp": [ -23.69773519163763, -15.331010452961673 ], "local_only_correct": 25, "jev_only_correct": 137, "both_correct": 328, "both_wrong": 84, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 409, "jev_correct": 490, "local_accuracy": 0.5842857142857143, "jev_accuracy": 0.7, "difference_pp": -11.571428571428571, "paired_cluster_bootstrap95_pp": [ -15.507496251874063, -7.406619385342794 ], "local_only_correct": 68, "jev_only_correct": 149, "both_correct": 341, "both_wrong": 142, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 72, "jev_correct": 97, "local_accuracy": 0.18, "jev_accuracy": 0.2425, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -11.0, -1.5 ], "local_only_correct": 36, "jev_only_correct": 61, "both_correct": 36, "both_wrong": 267, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en/reversed": { "n": 40, "local_correct": 38, "jev_correct": 40, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 38, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 38, "jev_correct": 40, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 38, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 34, "jev_correct": 39, "local_accuracy": 0.85, "jev_accuracy": 0.975, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -22.5, -2.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 34, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 37, "jev_correct": 39, "local_accuracy": 0.925, "jev_accuracy": 0.975, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 37, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 169, "jev_correct": 237, "local_accuracy": 0.4225, "jev_accuracy": 0.5925, "difference_pp": -17.0, "paired_cluster_bootstrap95_pp": [ -21.25625, -12.5 ], "local_only_correct": 15, "jev_only_correct": 83, "both_correct": 154, "both_wrong": 148, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 277, "jev_correct": 277, "local_accuracy": 0.9233333333333333, "jev_accuracy": 0.9233333333333333, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -2.0, 2.3333333333333335 ], "local_only_correct": 6, "jev_only_correct": 6, "both_correct": 271, "both_wrong": 17, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 307, "jev_correct": 377, "local_accuracy": 0.7675, "jev_accuracy": 0.9425, "difference_pp": -17.5, "paired_cluster_bootstrap95_pp": [ -21.5, -13.75 ], "local_only_correct": 4, "jev_only_correct": 74, "both_correct": 303, "both_wrong": 19, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 320, "jev_correct": 346, "local_accuracy": 0.8, "jev_accuracy": 0.865, "difference_pp": -6.5, "paired_cluster_bootstrap95_pp": [ -9.0, -4.0 ], "local_only_correct": 1, "jev_only_correct": 27, "both_correct": 319, "both_wrong": 53, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 335, "jev_correct": 360, "local_accuracy": 0.8375, "jev_accuracy": 0.9, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -9.25, -3.5 ], "local_only_correct": 6, "jev_only_correct": 31, "both_correct": 329, "both_wrong": 34, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 237, "jev_correct": 225, "local_accuracy": 0.79, "jev_accuracy": 0.75, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ 0.6583333333333338, 7.333333333333333 ], "local_only_correct": 20, "jev_only_correct": 8, "both_correct": 217, "both_wrong": 55, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 333, "jev_correct": 393, "local_accuracy": 0.8325, "jev_accuracy": 0.9825, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -18.5, -11.5 ], "local_only_correct": 2, "jev_only_correct": 62, "both_correct": 331, "both_wrong": 5, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 257, "jev_correct": 265, "local_accuracy": 0.6425, "jev_accuracy": 0.6625, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -5.5, 1.25 ], "local_only_correct": 20, "jev_only_correct": 28, "both_correct": 237, "both_wrong": 115, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 69, "jev_correct": 75, "local_accuracy": 0.69, "jev_accuracy": 0.75, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -13.0, 1.0 ], "local_only_correct": 3, "jev_only_correct": 9, "both_correct": 66, "both_wrong": 22, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 53, "jev_correct": 56, "local_accuracy": 0.53, "jev_accuracy": 0.56, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -12.0, 6.0 ], "local_only_correct": 9, "jev_only_correct": 12, "both_correct": 44, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 22, "jev_correct": 49, "local_accuracy": 0.22, "jev_accuracy": 0.49, "difference_pp": -27.0, "paired_cluster_bootstrap95_pp": [ -37.0, -16.0 ], "local_only_correct": 5, "jev_only_correct": 32, "both_correct": 17, "both_wrong": 46, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 43, "jev_correct": 48, "local_accuracy": 0.43, "jev_accuracy": 0.48, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -13.0, 3.0 ], "local_only_correct": 6, "jev_only_correct": 11, "both_correct": 37, "both_wrong": 46, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 79, "jev_correct": 79, "local_accuracy": 0.79, "jev_accuracy": 0.79, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -4.0, 4.0 ], "local_only_correct": 2, "jev_only_correct": 2, "both_correct": 77, "both_wrong": 19, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 6, "jev_correct": 6, "local_accuracy": 0.06, "jev_accuracy": 0.06, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -5.0, 5.0 ], "local_only_correct": 3, "jev_only_correct": 3, "both_correct": 3, "both_wrong": 91, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 61, "jev_correct": 64, "local_accuracy": 0.61, "jev_accuracy": 0.64, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -10.0, 4.0 ], "local_only_correct": 5, "jev_only_correct": 8, "both_correct": 56, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 41, "jev_correct": 37, "local_accuracy": 0.41, "jev_accuracy": 0.37, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -2.0, 10.0 ], "local_only_correct": 7, "jev_only_correct": 3, "both_correct": 34, "both_wrong": 56, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 6, "jev_correct": 16, "local_accuracy": 0.06, "jev_accuracy": 0.16, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -19.0, -2.0 ], "local_only_correct": 5, "jev_only_correct": 15, "both_correct": 1, "both_wrong": 79, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 29, "jev_correct": 34, "local_accuracy": 0.29, "jev_accuracy": 0.34, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -13.0, 2.0 ], "local_only_correct": 5, "jev_only_correct": 10, "both_correct": 24, "both_wrong": 61, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 86, "jev_correct": 83, "local_accuracy": 0.86, "jev_accuracy": 0.83, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -3.0, 9.0 ], "local_only_correct": 6, "jev_only_correct": 3, "both_correct": 80, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 22, "jev_correct": 21, "local_accuracy": 0.22, "jev_accuracy": 0.21, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -6.0, 8.0 ], "local_only_correct": 8, "jev_only_correct": 7, "both_correct": 14, "both_wrong": 71, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 44, "jev_correct": 51, "local_accuracy": 0.44, "jev_accuracy": 0.51, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -15.0, 1.0 ], "local_only_correct": 5, "jev_only_correct": 12, "both_correct": 39, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 61, "jev_correct": 65, "local_accuracy": 0.61, "jev_accuracy": 0.65, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -12.0, 4.0 ], "local_only_correct": 6, "jev_only_correct": 10, "both_correct": 55, "both_wrong": 29, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 40, "jev_correct": 39, "local_accuracy": 0.4, "jev_accuracy": 0.39, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 9.0 ], "local_only_correct": 8, "jev_only_correct": 7, "both_correct": 32, "both_wrong": 53, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 9, "jev_correct": 9, "local_accuracy": 0.09, "jev_accuracy": 0.09, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -4.0, 4.0 ], "local_only_correct": 2, "jev_only_correct": 2, "both_correct": 7, "both_wrong": 89, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 58, "jev_correct": 59, "local_accuracy": 0.58, "jev_accuracy": 0.59, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 4.0 ], "local_only_correct": 3, "jev_only_correct": 4, "both_correct": 55, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 86, "jev_correct": 91, "local_accuracy": 0.86, "jev_accuracy": 0.91, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -10.0, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 85, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 93, "jev_correct": 88, "local_accuracy": 0.93, "jev_accuracy": 0.88, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -1.0, 11.0 ], "local_only_correct": 7, "jev_only_correct": 2, "both_correct": 86, "both_wrong": 5, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 32, "jev_correct": 48, "local_accuracy": 0.32, "jev_accuracy": 0.48, "difference_pp": -16.0, "paired_cluster_bootstrap95_pp": [ -25.0, -8.0 ], "local_only_correct": 3, "jev_only_correct": 19, "both_correct": 29, "both_wrong": 49, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2524, "jev_correct": 2550, "local_accuracy": 0.8561736770691994, "jev_accuracy": 0.8649932157394844, "difference_pp": -0.8819538670284939, "paired_cluster_bootstrap95_pp": [ -1.8995929443690638, 0.20352781546811397 ], "local_only_correct": 109, "jev_only_correct": 135, "both_correct": 2415, "both_wrong": 289, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2424, "jev_correct": 2412, "local_accuracy": 0.8222523744911805, "jev_accuracy": 0.8181818181818182, "difference_pp": 0.40705563093622793, "paired_cluster_bootstrap95_pp": [ -0.9158751696065129, 1.7299864314789688 ], "local_only_correct": 200, "jev_only_correct": 188, "both_correct": 2224, "both_wrong": 336, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 819, "jev_correct": 843, "local_accuracy": 0.6982097186700768, "jev_accuracy": 0.7186700767263428, "difference_pp": -2.0460358056265986, "paired_cluster_bootstrap95_pp": [ -5.5615745448682965, 1.3514658726523128 ], "local_only_correct": 174, "jev_only_correct": 198, "both_correct": 645, "both_wrong": 156, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "intern-decision-4b": { "fresh/news": { "n": 200, "local_correct": 176, "jev_correct": 178, "local_accuracy": 0.88, "jev_accuracy": 0.89, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 2.0 ], "local_only_correct": 4, "jev_only_correct": 6, "both_correct": 172, "both_wrong": 18, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 96, "jev_correct": 120, "local_accuracy": 0.48, "jev_accuracy": 0.6, "difference_pp": -12.0, "paired_cluster_bootstrap95_pp": [ -18.5, -5.5 ], "local_only_correct": 12, "jev_only_correct": 36, "both_correct": 84, "both_wrong": 68, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 188, "jev_correct": 196, "local_accuracy": 0.94, "jev_accuracy": 0.98, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -7.0, -1.5 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 188, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 183, "jev_correct": 179, "local_accuracy": 0.915, "jev_accuracy": 0.895, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -1.0, 5.5 ], "local_only_correct": 7, "jev_only_correct": 3, "both_correct": 176, "both_wrong": 14, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 10, "jev_correct": 11, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9166666666666666, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 10, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 9, "jev_correct": 10, "local_accuracy": 0.75, "jev_accuracy": 0.8333333333333334, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 9, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 92, "jev_correct": 91, "local_accuracy": 0.92, "jev_accuracy": 0.91, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -3.0, 6.0 ], "local_only_correct": 3, "jev_only_correct": 2, "both_correct": 89, "both_wrong": 6, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 47, "jev_correct": 48, "local_accuracy": 0.47, "jev_accuracy": 0.48, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -9.0, 7.0 ], "local_only_correct": 8, "jev_only_correct": 9, "both_correct": 39, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 0, "jev_correct": 89, "local_accuracy": 0.0, "jev_accuracy": 0.89, "difference_pp": -89.0, "paired_cluster_bootstrap95_pp": [ -95.0, -82.0 ], "local_only_correct": 0, "jev_only_correct": 89, "both_correct": 0, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 82, "jev_correct": 81, "local_accuracy": 0.7387387387387387, "jev_accuracy": 0.7297297297297297, "difference_pp": 0.9009009009009009, "paired_cluster_bootstrap95_pp": [ -8.108108108108109, 9.90990990990991 ], "local_only_correct": 12, "jev_only_correct": 11, "both_correct": 70, "both_wrong": 18, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 71, "jev_correct": 70, "local_accuracy": 0.9861111111111112, "jev_accuracy": 0.9722222222222222, "difference_pp": 1.3888888888888888, "paired_cluster_bootstrap95_pp": [ -2.7777777777777777, 6.944444444444445 ], "local_only_correct": 2, "jev_only_correct": 1, "both_correct": 69, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 19, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 0.9473684210526315, "difference_pp": 5.263157894736842, "paired_cluster_bootstrap95_pp": [ 0.0, 15.789473684210526 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9090909090909091, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.8235294117647058, "jev_accuracy": 0.9411764705882353, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 19, "jev_correct": 21, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9545454545454546, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -22.727272727272727, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9444444444444444, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -22.22222222222222, 11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.6875, "jev_accuracy": 0.9375, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -43.90624999999999, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 18, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 0.9444444444444444, "difference_pp": 5.555555555555555, "paired_cluster_bootstrap95_pp": [ 0.0, 16.666666666666668 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 19, "jev_correct": 18, "local_accuracy": 0.95, "jev_accuracy": 0.9, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ 0.0, 15.0 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.6190476190476191, "jev_accuracy": 0.8571428571428571, "difference_pp": -23.809523809523807, "paired_cluster_bootstrap95_pp": [ -47.61904761904762, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 12, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.8571428571428571, "jev_accuracy": 1.0, "difference_pp": -14.285714285714285, "paired_cluster_bootstrap95_pp": [ -35.714285714285715, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 14, "jev_correct": 14, "local_accuracy": 0.875, "jev_accuracy": 0.875, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -18.75, 18.75 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.75, "jev_accuracy": 0.875, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -37.5, 12.5 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7777777777777778, "jev_accuracy": 0.9444444444444444, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.75, "jev_accuracy": 0.9375, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -43.75, 6.25 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8421052631578947, "jev_accuracy": 0.8947368421052632, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -26.31578947368421, 10.526315789473685 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 11, "jev_correct": 19, "local_accuracy": 0.55, "jev_accuracy": 0.95, "difference_pp": -40.0, "paired_cluster_bootstrap95_pp": [ -60.0, -20.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 11, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 20, "jev_correct": 20, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 17, "jev_correct": 16, "local_accuracy": 0.85, "jev_accuracy": 0.8, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -10.0, 20.0 ], "local_only_correct": 2, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 13.636363636363637 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 557, "jev_correct": 629, "local_accuracy": 0.6188888888888889, "jev_accuracy": 0.6988888888888889, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -11.167827978686509, -5.110124504327805 ], "local_only_correct": 59, "jev_only_correct": 131, "both_correct": 498, "both_wrong": 212, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 426, "jev_correct": 465, "local_accuracy": 0.7421602787456446, "jev_accuracy": 0.8101045296167247, "difference_pp": -6.794425087108014, "paired_cluster_bootstrap95_pp": [ -10.278745644599303, -3.484320557491289 ], "local_only_correct": 31, "jev_only_correct": 70, "both_correct": 395, "both_wrong": 78, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 293, "jev_correct": 490, "local_accuracy": 0.4185714285714286, "jev_accuracy": 0.7, "difference_pp": -28.142857142857142, "paired_cluster_bootstrap95_pp": [ -32.04036863006265, -24.113475177304963 ], "local_only_correct": 33, "jev_only_correct": 230, "both_correct": 260, "both_wrong": 177, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 155, "jev_correct": 97, "local_accuracy": 0.3875, "jev_accuracy": 0.2425, "difference_pp": 14.499999999999998, "paired_cluster_bootstrap95_pp": [ 9.0, 19.75 ], "local_only_correct": 90, "jev_only_correct": 32, "both_correct": 65, "both_wrong": 213, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 39, "jev_correct": 40, "local_accuracy": 0.975, "jev_accuracy": 1.0, "difference_pp": -2.5, "paired_cluster_bootstrap95_pp": [ -7.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 39, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 39, "jev_correct": 40, "local_accuracy": 0.975, "jev_accuracy": 1.0, "difference_pp": -2.5, "paired_cluster_bootstrap95_pp": [ -7.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 39, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 37, "jev_correct": 39, "local_accuracy": 0.925, "jev_accuracy": 0.975, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 37, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 39, "jev_correct": 39, "local_accuracy": 0.975, "jev_accuracy": 0.975, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 39, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 215, "jev_correct": 237, "local_accuracy": 0.5375, "jev_accuracy": 0.5925, "difference_pp": -5.5, "paired_cluster_bootstrap95_pp": [ -9.75, -1.25 ], "local_only_correct": 27, "jev_only_correct": 49, "both_correct": 188, "both_wrong": 136, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 273, "jev_correct": 277, "local_accuracy": 0.91, "jev_accuracy": 0.9233333333333333, "difference_pp": -1.3333333333333335, "paired_cluster_bootstrap95_pp": [ -4.0, 1.0 ], "local_only_correct": 5, "jev_only_correct": 9, "both_correct": 268, "both_wrong": 18, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 332, "jev_correct": 377, "local_accuracy": 0.83, "jev_accuracy": 0.9425, "difference_pp": -11.25, "paired_cluster_bootstrap95_pp": [ -14.75625, -8.0 ], "local_only_correct": 5, "jev_only_correct": 50, "both_correct": 327, "both_wrong": 18, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 355, "jev_correct": 346, "local_accuracy": 0.8875, "jev_accuracy": 0.865, "difference_pp": 2.25, "paired_cluster_bootstrap95_pp": [ -0.25, 4.75 ], "local_only_correct": 18, "jev_only_correct": 9, "both_correct": 337, "both_wrong": 36, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 368, "jev_correct": 360, "local_accuracy": 0.92, "jev_accuracy": 0.9, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ 0.25, 3.75 ], "local_only_correct": 10, "jev_only_correct": 2, "both_correct": 358, "both_wrong": 30, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 234, "jev_correct": 225, "local_accuracy": 0.78, "jev_accuracy": 0.75, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ 0.0, 6.0 ], "local_only_correct": 16, "jev_only_correct": 7, "both_correct": 218, "both_wrong": 59, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 378, "jev_correct": 393, "local_accuracy": 0.945, "jev_accuracy": 0.9825, "difference_pp": -3.75, "paired_cluster_bootstrap95_pp": [ -5.75, -1.75 ], "local_only_correct": 1, "jev_only_correct": 16, "both_correct": 377, "both_wrong": 6, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 309, "jev_correct": 265, "local_accuracy": 0.7725, "jev_accuracy": 0.6625, "difference_pp": 11.0, "paired_cluster_bootstrap95_pp": [ 7.0, 15.25 ], "local_only_correct": 58, "jev_only_correct": 14, "both_correct": 251, "both_wrong": 77, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 74, "jev_correct": 75, "local_accuracy": 0.74, "jev_accuracy": 0.75, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -8.0, 5.0 ], "local_only_correct": 5, "jev_only_correct": 6, "both_correct": 69, "both_wrong": 20, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 42, "jev_correct": 56, "local_accuracy": 0.42, "jev_accuracy": 0.56, "difference_pp": -14.000000000000002, "paired_cluster_bootstrap95_pp": [ -22.0, -6.0 ], "local_only_correct": 3, "jev_only_correct": 17, "both_correct": 39, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 50, "jev_correct": 49, "local_accuracy": 0.5, "jev_accuracy": 0.49, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 7.0 ], "local_only_correct": 5, "jev_only_correct": 4, "both_correct": 45, "both_wrong": 46, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 38, "jev_correct": 48, "local_accuracy": 0.38, "jev_accuracy": 0.48, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -17.0, -4.0 ], "local_only_correct": 1, "jev_only_correct": 11, "both_correct": 37, "both_wrong": 51, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 74, "jev_correct": 79, "local_accuracy": 0.74, "jev_accuracy": 0.79, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -10.024999999999999, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 73, "both_wrong": 20, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 7, "jev_correct": 6, "local_accuracy": 0.07, "jev_accuracy": 0.06, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 3, "both_correct": 3, "both_wrong": 90, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 61, "jev_correct": 64, "local_accuracy": 0.61, "jev_accuracy": 0.64, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -10.0, 4.0 ], "local_only_correct": 6, "jev_only_correct": 9, "both_correct": 55, "both_wrong": 30, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 38, "jev_correct": 37, "local_accuracy": 0.38, "jev_accuracy": 0.37, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 8.0 ], "local_only_correct": 6, "jev_only_correct": 5, "both_correct": 32, "both_wrong": 57, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 11, "jev_correct": 16, "local_accuracy": 0.11, "jev_accuracy": 0.16, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -13.0, 2.0 ], "local_only_correct": 5, "jev_only_correct": 10, "both_correct": 6, "both_wrong": 79, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 39, "jev_correct": 34, "local_accuracy": 0.39, "jev_accuracy": 0.34, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -5.0, 15.0 ], "local_only_correct": 15, "jev_only_correct": 10, "both_correct": 24, "both_wrong": 51, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 87, "jev_correct": 83, "local_accuracy": 0.87, "jev_accuracy": 0.83, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -1.0, 9.0 ], "local_only_correct": 5, "jev_only_correct": 1, "both_correct": 82, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 19, "jev_correct": 21, "local_accuracy": 0.19, "jev_accuracy": 0.21, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -11.0, 7.0 ], "local_only_correct": 10, "jev_only_correct": 12, "both_correct": 9, "both_wrong": 69, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 52, "jev_correct": 51, "local_accuracy": 0.52, "jev_accuracy": 0.51, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 8.0 ], "local_only_correct": 6, "jev_only_correct": 5, "both_correct": 46, "both_wrong": 43, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 54, "jev_correct": 65, "local_accuracy": 0.54, "jev_accuracy": 0.65, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -17.0, -5.0 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 54, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 35, "jev_correct": 39, "local_accuracy": 0.35, "jev_accuracy": 0.39, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -10.0, 2.0 ], "local_only_correct": 3, "jev_only_correct": 7, "both_correct": 32, "both_wrong": 58, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 8, "jev_correct": 9, "local_accuracy": 0.08, "jev_accuracy": 0.09, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -6.0, 4.0 ], "local_only_correct": 3, "jev_only_correct": 4, "both_correct": 5, "both_wrong": 88, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 58, "jev_correct": 59, "local_accuracy": 0.58, "jev_accuracy": 0.59, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 5.0 ], "local_only_correct": 5, "jev_only_correct": 6, "both_correct": 53, "both_wrong": 36, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 94, "jev_correct": 91, "local_accuracy": 0.94, "jev_accuracy": 0.91, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -4.0, 9.024999999999864 ], "local_only_correct": 7, "jev_only_correct": 4, "both_correct": 87, "both_wrong": 2, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 0, "jev_correct": 88, "local_accuracy": 0.0, "jev_accuracy": 0.88, "difference_pp": -88.0, "paired_cluster_bootstrap95_pp": [ -94.0, -81.0 ], "local_only_correct": 0, "jev_only_correct": 88, "both_correct": 0, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 48, "jev_correct": 48, "local_accuracy": 0.48, "jev_accuracy": 0.48, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -8.0, 9.0 ], "local_only_correct": 9, "jev_only_correct": 9, "both_correct": 39, "both_wrong": 43, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2448, "jev_correct": 2550, "local_accuracy": 0.830393487109905, "jev_accuracy": 0.8649932157394844, "difference_pp": -3.4599728629579376, "paired_cluster_bootstrap95_pp": [ -4.511533242876526, -2.37449118046133 ], "local_only_correct": 70, "jev_only_correct": 172, "both_correct": 2378, "both_wrong": 328, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2126, "jev_correct": 2412, "local_accuracy": 0.7211668928086838, "jev_accuracy": 0.8181818181818182, "difference_pp": -9.701492537313433, "paired_cluster_bootstrap95_pp": [ -11.025271370420624, -8.276797829036635 ], "local_only_correct": 76, "jev_only_correct": 362, "both_correct": 2050, "both_wrong": 460, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 802, "jev_correct": 843, "local_accuracy": 0.6837169650468883, "jev_accuracy": 0.7186700767263428, "difference_pp": -3.495311167945439, "paired_cluster_bootstrap95_pp": [ -5.800331049996463, -1.2009603554058197 ], "local_only_correct": 89, "jev_only_correct": 130, "both_correct": 713, "both_wrong": 241, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "jev-omni-12b-nf4": { "fresh/news": { "n": 200, "local_correct": 170, "jev_correct": 178, "local_accuracy": 0.85, "jev_accuracy": 0.89, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -8.0, -0.5 ], "local_only_correct": 3, "jev_only_correct": 11, "both_correct": 167, "both_wrong": 19, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 106, "jev_correct": 120, "local_accuracy": 0.53, "jev_accuracy": 0.6, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -12.5, -1.5 ], "local_only_correct": 9, "jev_only_correct": 23, "both_correct": 97, "both_wrong": 71, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 193, "jev_correct": 196, "local_accuracy": 0.965, "jev_accuracy": 0.98, "difference_pp": -1.5, "paired_cluster_bootstrap95_pp": [ -4.0, 0.5 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 192, "both_wrong": 3, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 176, "jev_correct": 179, "local_accuracy": 0.88, "jev_accuracy": 0.895, "difference_pp": -1.5, "paired_cluster_bootstrap95_pp": [ -5.0, 2.5 ], "local_only_correct": 6, "jev_only_correct": 9, "both_correct": 170, "both_wrong": 15, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 11, "jev_correct": 11, "local_accuracy": 0.9166666666666666, "jev_accuracy": 0.9166666666666666, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -25.0, 25.0 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 10, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 12, "jev_correct": 10, "local_accuracy": 1.0, "jev_accuracy": 0.8333333333333334, "difference_pp": 16.666666666666664, "paired_cluster_bootstrap95_pp": [ 0.0, 41.666666666666664 ], "local_only_correct": 2, "jev_only_correct": 0, "both_correct": 10, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 11, "jev_correct": 12, "local_accuracy": 0.9166666666666666, "jev_accuracy": 1.0, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 11, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 85, "jev_correct": 91, "local_accuracy": 0.85, "jev_accuracy": 0.91, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -11.0, -2.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 85, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 46, "jev_correct": 48, "local_accuracy": 0.46, "jev_accuracy": 0.48, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -10.0, 6.0 ], "local_only_correct": 7, "jev_only_correct": 9, "both_correct": 39, "both_wrong": 45, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 52, "jev_correct": 89, "local_accuracy": 0.52, "jev_accuracy": 0.89, "difference_pp": -37.0, "paired_cluster_bootstrap95_pp": [ -47.0, -27.0 ], "local_only_correct": 2, "jev_only_correct": 39, "both_correct": 50, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 81, "jev_correct": 81, "local_accuracy": 0.7297297297297297, "jev_accuracy": 0.7297297297297297, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -8.108108108108109, 8.108108108108109 ], "local_only_correct": 11, "jev_only_correct": 11, "both_correct": 70, "both_wrong": 19, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 69, "jev_correct": 70, "local_accuracy": 0.9583333333333334, "jev_accuracy": 0.9722222222222222, "difference_pp": -1.3888888888888888, "paired_cluster_bootstrap95_pp": [ -6.944444444444445, 2.7777777777777777 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 68, "both_wrong": 1, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 19, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 0.9473684210526315, "difference_pp": 5.263157894736842, "paired_cluster_bootstrap95_pp": [ 0.0, 15.789473684210526 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 22, "jev_correct": 20, "local_accuracy": 1.0, "jev_accuracy": 0.9090909090909091, "difference_pp": 9.090909090909092, "paired_cluster_bootstrap95_pp": [ 0.0, 22.727272727272727 ], "local_only_correct": 2, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.8235294117647058, "jev_accuracy": 0.9411764705882353, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 20, "jev_correct": 21, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9545454545454546, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 20, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/expense_category": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9444444444444444, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -22.22222222222222, 11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 15, "jev_correct": 15, "local_accuracy": 0.9375, "jev_accuracy": 0.9375, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -18.75, 18.75 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 18, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 0.9444444444444444, "difference_pp": 5.555555555555555, "paired_cluster_bootstrap95_pp": [ 0.0, 16.666666666666668 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 19, "jev_correct": 18, "local_accuracy": 0.95, "jev_accuracy": 0.9, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ 0.0, 15.0 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.8571428571428571, "jev_accuracy": 0.8571428571428571, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -23.80952380952381, 23.80952380952381 ], "local_only_correct": 3, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 13, "jev_correct": 14, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -21.428571428571427, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 13, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 13, "jev_correct": 14, "local_accuracy": 0.8125, "jev_accuracy": 0.875, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -25.0, 12.5 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.6875, "jev_accuracy": 0.875, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -43.75, 6.25 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9444444444444444, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -25.0, 12.5 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8421052631578947, "jev_accuracy": 0.8947368421052632, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -26.31578947368421, 15.789473684210526 ], "local_only_correct": 2, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 17, "jev_correct": 19, "local_accuracy": 0.85, "jev_accuracy": 0.95, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -30.0, 10.0 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 16, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 18, "jev_correct": 20, "local_accuracy": 0.9, "jev_accuracy": 1.0, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 18, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 17, "jev_correct": 16, "local_accuracy": 0.85, "jev_accuracy": 0.8, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 25.0 ], "local_only_correct": 3, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 21, "jev_correct": 20, "local_accuracy": 0.9545454545454546, "jev_accuracy": 0.9090909090909091, "difference_pp": 4.545454545454546, "paired_cluster_bootstrap95_pp": [ -9.090909090909092, 22.727272727272727 ], "local_only_correct": 2, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 573, "jev_correct": 629, "local_accuracy": 0.6366666666666667, "jev_accuracy": 0.6988888888888889, "difference_pp": -6.222222222222222, "paired_cluster_bootstrap95_pp": [ -11.533742331288343, -1.045348760980728 ], "local_only_correct": 87, "jev_only_correct": 143, "both_correct": 486, "both_wrong": 184, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 417, "jev_correct": 465, "local_accuracy": 0.7264808362369338, "jev_accuracy": 0.8101045296167247, "difference_pp": -8.362369337979095, "paired_cluster_bootstrap95_pp": [ -11.846689895470384, -5.052264808362369 ], "local_only_correct": 26, "jev_only_correct": 74, "both_correct": 391, "both_wrong": 83, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 366, "jev_correct": 490, "local_accuracy": 0.5228571428571429, "jev_accuracy": 0.7, "difference_pp": -17.71428571428571, "paired_cluster_bootstrap95_pp": [ -21.75792507204611, -13.75308975982882 ], "local_only_correct": 55, "jev_only_correct": 179, "both_correct": 311, "both_wrong": 155, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 72, "jev_correct": 97, "local_accuracy": 0.18, "jev_accuracy": 0.2425, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -11.0, -1.5 ], "local_only_correct": 35, "jev_only_correct": 60, "both_correct": 37, "both_wrong": 268, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 26, "jev_correct": 28, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -17.857142857142858, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 26, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 26, "jev_correct": 28, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -17.857142857142858, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 26, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 40, "jev_correct": 40, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 40, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en": { "n": 40, "local_correct": 40, "jev_correct": 40, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 40, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 40, "jev_correct": 39, "local_accuracy": 1.0, "jev_accuracy": 0.975, "difference_pp": 2.5, "paired_cluster_bootstrap95_pp": [ 0.0, 7.5 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 39, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 39, "jev_correct": 39, "local_accuracy": 0.975, "jev_accuracy": 0.975, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -7.5, 7.5 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 38, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 215, "jev_correct": 237, "local_accuracy": 0.5375, "jev_accuracy": 0.5925, "difference_pp": -5.5, "paired_cluster_bootstrap95_pp": [ -9.00625, -2.0 ], "local_only_correct": 18, "jev_only_correct": 40, "both_correct": 197, "both_wrong": 145, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 255, "jev_correct": 277, "local_accuracy": 0.85, "jev_accuracy": 0.9233333333333333, "difference_pp": -7.333333333333333, "paired_cluster_bootstrap95_pp": [ -11.0, -3.6666666666666665 ], "local_only_correct": 6, "jev_only_correct": 28, "both_correct": 249, "both_wrong": 17, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 358, "jev_correct": 377, "local_accuracy": 0.895, "jev_accuracy": 0.9425, "difference_pp": -4.75, "paired_cluster_bootstrap95_pp": [ -7.0, -2.75 ], "local_only_correct": 1, "jev_only_correct": 20, "both_correct": 357, "both_wrong": 22, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 331, "jev_correct": 346, "local_accuracy": 0.8275, "jev_accuracy": 0.865, "difference_pp": -3.75, "paired_cluster_bootstrap95_pp": [ -6.0, -1.5 ], "local_only_correct": 3, "jev_only_correct": 18, "both_correct": 328, "both_wrong": 51, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 348, "jev_correct": 360, "local_accuracy": 0.87, "jev_accuracy": 0.9, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -5.75, -0.25 ], "local_only_correct": 10, "jev_only_correct": 22, "both_correct": 338, "both_wrong": 30, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 216, "jev_correct": 225, "local_accuracy": 0.72, "jev_accuracy": 0.75, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -6.333333333333333, 0.3333333333333333 ], "local_only_correct": 9, "jev_only_correct": 18, "both_correct": 207, "both_wrong": 66, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 384, "jev_correct": 393, "local_accuracy": 0.96, "jev_accuracy": 0.9825, "difference_pp": -2.25, "paired_cluster_bootstrap95_pp": [ -4.0, -0.75 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 383, "both_wrong": 6, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 293, "jev_correct": 265, "local_accuracy": 0.7325, "jev_accuracy": 0.6625, "difference_pp": 7.000000000000001, "paired_cluster_bootstrap95_pp": [ 3.5, 10.5 ], "local_only_correct": 41, "jev_only_correct": 13, "both_correct": 252, "both_wrong": 94, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 75, "jev_correct": 75, "local_accuracy": 0.75, "jev_accuracy": 0.75, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -5.0, 5.0 ], "local_only_correct": 4, "jev_only_correct": 4, "both_correct": 71, "both_wrong": 21, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 46, "jev_correct": 56, "local_accuracy": 0.46, "jev_accuracy": 0.56, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -17.0, -3.0 ], "local_only_correct": 2, "jev_only_correct": 12, "both_correct": 44, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 49, "jev_correct": 49, "local_accuracy": 0.49, "jev_accuracy": 0.49, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -5.0, 5.0 ], "local_only_correct": 3, "jev_only_correct": 3, "both_correct": 46, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 48, "jev_correct": 48, "local_accuracy": 0.48, "jev_accuracy": 0.48, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -6.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 4, "both_correct": 44, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 74, "jev_correct": 79, "local_accuracy": 0.74, "jev_accuracy": 0.79, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -10.0, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 73, "both_wrong": 20, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 5, "jev_correct": 6, "local_accuracy": 0.05, "jev_accuracy": 0.06, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 3.0 ], "local_only_correct": 2, "jev_only_correct": 3, "both_correct": 3, "both_wrong": 92, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 61, "jev_correct": 64, "local_accuracy": 0.61, "jev_accuracy": 0.64, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -9.0, 3.0 ], "local_only_correct": 4, "jev_only_correct": 7, "both_correct": 57, "both_wrong": 32, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 39, "jev_correct": 37, "local_accuracy": 0.39, "jev_accuracy": 0.37, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -6.0, 10.0 ], "local_only_correct": 10, "jev_only_correct": 8, "both_correct": 29, "both_wrong": 53, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.13, "jev_accuracy": 0.16, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -9.0, 3.0 ], "local_only_correct": 4, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 80, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 29, "jev_correct": 34, "local_accuracy": 0.29, "jev_accuracy": 0.34, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 4.0 ], "local_only_correct": 9, "jev_only_correct": 14, "both_correct": 20, "both_wrong": 57, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 83, "jev_correct": 83, "local_accuracy": 0.83, "jev_accuracy": 0.83, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -9.0, 9.0 ], "local_only_correct": 10, "jev_only_correct": 10, "both_correct": 73, "both_wrong": 7, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 23, "jev_correct": 21, "local_accuracy": 0.23, "jev_accuracy": 0.21, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -6.0, 10.0 ], "local_only_correct": 9, "jev_only_correct": 7, "both_correct": 14, "both_wrong": 70, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 45, "jev_correct": 51, "local_accuracy": 0.45, "jev_accuracy": 0.51, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -13.0, 1.0 ], "local_only_correct": 4, "jev_only_correct": 10, "both_correct": 41, "both_wrong": 45, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 52, "jev_correct": 65, "local_accuracy": 0.52, "jev_accuracy": 0.65, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -20.0, -7.0 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 52, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 38, "jev_correct": 39, "local_accuracy": 0.38, "jev_accuracy": 0.39, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -8.0, 6.0 ], "local_only_correct": 6, "jev_only_correct": 7, "both_correct": 32, "both_wrong": 55, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 11, "jev_correct": 9, "local_accuracy": 0.11, "jev_accuracy": 0.09, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -4.0, 8.0 ], "local_only_correct": 6, "jev_only_correct": 4, "both_correct": 5, "both_wrong": 85, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 58, "jev_correct": 59, "local_accuracy": 0.58, "jev_accuracy": 0.59, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 5.0 ], "local_only_correct": 5, "jev_only_correct": 6, "both_correct": 53, "both_wrong": 36, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 82, "jev_correct": 91, "local_accuracy": 0.82, "jev_accuracy": 0.91, "difference_pp": -9.0, "paired_cluster_bootstrap95_pp": [ -15.0, -4.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 82, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 52, "jev_correct": 88, "local_accuracy": 0.52, "jev_accuracy": 0.88, "difference_pp": -36.0, "paired_cluster_bootstrap95_pp": [ -45.0, -27.0 ], "local_only_correct": 0, "jev_only_correct": 36, "both_correct": 52, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 44, "jev_correct": 48, "local_accuracy": 0.44, "jev_accuracy": 0.48, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -12.0, 3.0 ], "local_only_correct": 5, "jev_only_correct": 9, "both_correct": 39, "both_wrong": 47, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2365, "jev_correct": 2550, "local_accuracy": 0.8022388059701493, "jev_accuracy": 0.8649932157394844, "difference_pp": -6.275440976933515, "paired_cluster_bootstrap95_pp": [ -7.462686567164179, -5.020352781546811 ], "local_only_correct": 73, "jev_only_correct": 258, "both_correct": 2292, "both_wrong": 325, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2055, "jev_correct": 2412, "local_accuracy": 0.6970827679782904, "jev_accuracy": 0.8181818181818182, "difference_pp": -12.109905020352782, "paired_cluster_bootstrap95_pp": [ -13.53459972862958, -10.583446404341927 ], "local_only_correct": 92, "jev_only_correct": 449, "both_correct": 1963, "both_wrong": 444, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 743, "jev_correct": 843, "local_accuracy": 0.6334185848252344, "jev_accuracy": 0.7186700767263428, "difference_pp": -8.525149190110827, "paired_cluster_bootstrap95_pp": [ -11.333753296021534, -5.582237383325761 ], "local_only_correct": 64, "jev_only_correct": 164, "both_correct": 679, "both_wrong": 266, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "jevk5-2b": { "fresh/news": { "n": 200, "local_correct": 170, "jev_correct": 178, "local_accuracy": 0.85, "jev_accuracy": 0.89, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -8.0, 0.0 ], "local_only_correct": 4, "jev_only_correct": 12, "both_correct": 166, "both_wrong": 18, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 124, "jev_correct": 120, "local_accuracy": 0.62, "jev_accuracy": 0.6, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -3.0, 7.0 ], "local_only_correct": 14, "jev_only_correct": 10, "both_correct": 110, "both_wrong": 66, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 141, "jev_correct": 196, "local_accuracy": 0.705, "jev_accuracy": 0.98, "difference_pp": -27.500000000000004, "paired_cluster_bootstrap95_pp": [ -34.0, -21.5 ], "local_only_correct": 1, "jev_only_correct": 56, "both_correct": 140, "both_wrong": 3, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 147, "jev_correct": 179, "local_accuracy": 0.735, "jev_accuracy": 0.895, "difference_pp": -16.0, "paired_cluster_bootstrap95_pp": [ -23.5, -8.0 ], "local_only_correct": 14, "jev_only_correct": 46, "both_correct": 133, "both_wrong": 7, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 9, "jev_correct": 11, "local_accuracy": 0.75, "jev_accuracy": 0.9166666666666666, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 9, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 5, "jev_correct": 10, "local_accuracy": 0.4166666666666667, "jev_accuracy": 0.8333333333333334, "difference_pp": -41.66666666666667, "paired_cluster_bootstrap95_pp": [ -83.33333333333333, 0.0 ], "local_only_correct": 2, "jev_only_correct": 7, "both_correct": 3, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 11, "jev_correct": 12, "local_accuracy": 0.9166666666666666, "jev_accuracy": 1.0, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 11, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 8, "jev_correct": 12, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -58.333333333333336, -8.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 8, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 85, "jev_correct": 91, "local_accuracy": 0.85, "jev_accuracy": 0.91, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -12.0, -1.0 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 84, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 46, "jev_correct": 48, "local_accuracy": 0.46, "jev_accuracy": 0.48, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -10.0, 7.0 ], "local_only_correct": 8, "jev_only_correct": 10, "both_correct": 38, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 61, "jev_correct": 89, "local_accuracy": 0.61, "jev_accuracy": 0.89, "difference_pp": -28.000000000000004, "paired_cluster_bootstrap95_pp": [ -37.0, -19.0 ], "local_only_correct": 1, "jev_only_correct": 29, "both_correct": 60, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 68, "jev_correct": 81, "local_accuracy": 0.6126126126126126, "jev_accuracy": 0.7297297297297297, "difference_pp": -11.711711711711711, "paired_cluster_bootstrap95_pp": [ -21.62162162162162, -1.8018018018018018 ], "local_only_correct": 10, "jev_only_correct": 23, "both_correct": 58, "both_wrong": 20, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 57, "jev_correct": 70, "local_accuracy": 0.7916666666666666, "jev_accuracy": 0.9722222222222222, "difference_pp": -18.055555555555554, "paired_cluster_bootstrap95_pp": [ -29.166666666666668, -8.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 57, "both_wrong": 2, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8421052631578947, "jev_accuracy": 0.9473684210526315, "difference_pp": -10.526315789473683, "paired_cluster_bootstrap95_pp": [ -26.31578947368421, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.8125, "jev_accuracy": 1.0, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 14, "jev_correct": 20, "local_accuracy": 0.6363636363636364, "jev_accuracy": 0.9090909090909091, "difference_pp": -27.27272727272727, "paired_cluster_bootstrap95_pp": [ -54.54545454545455, 0.0 ], "local_only_correct": 2, "jev_only_correct": 8, "both_correct": 12, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 11, "jev_correct": 18, "local_accuracy": 0.6111111111111112, "jev_accuracy": 1.0, "difference_pp": -38.88888888888889, "paired_cluster_bootstrap95_pp": [ -61.111111111111114, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 11, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.7058823529411765, "jev_accuracy": 0.9411764705882353, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 9, "jev_correct": 15, "local_accuracy": 0.6, "jev_accuracy": 1.0, "difference_pp": -40.0, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -20.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 9, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7777777777777778, "jev_accuracy": 1.0, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 17, "jev_correct": 21, "local_accuracy": 0.7727272727272727, "jev_accuracy": 0.9545454545454546, "difference_pp": -18.181818181818183, "paired_cluster_bootstrap95_pp": [ -36.36363636363637, -4.545454545454546 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 17, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -38.888888888888886, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6111111111111112, "jev_accuracy": 0.9444444444444444, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -61.111111111111114, -5.555555555555555 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5, "jev_accuracy": 0.9375, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9444444444444444, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8, "jev_accuracy": 0.9, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -30.0, 10.0 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 10, "jev_correct": 18, "local_accuracy": 0.47619047619047616, "jev_accuracy": 0.8571428571428571, "difference_pp": -38.095238095238095, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -9.523809523809524 ], "local_only_correct": 2, "jev_only_correct": 10, "both_correct": 8, "both_wrong": 1, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.5625, "jev_accuracy": 1.0, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 22, "jev_correct": 23, "local_accuracy": 0.9565217391304348, "jev_accuracy": 1.0, "difference_pp": -4.3478260869565215, "paired_cluster_bootstrap95_pp": [ -13.043478260869565, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 22, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 8, "jev_correct": 14, "local_accuracy": 0.5, "jev_accuracy": 0.875, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 8, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 6, "jev_correct": 14, "local_accuracy": 0.375, "jev_accuracy": 0.875, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -81.25, -18.75 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 5, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.8125, "jev_accuracy": 1.0, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7777777777777778, "jev_accuracy": 1.0, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5, "jev_accuracy": 0.9444444444444444, "difference_pp": -44.44444444444444, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -22.22222222222222 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5, "jev_accuracy": 0.9375, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5333333333333333, "jev_accuracy": 1.0, "difference_pp": -46.666666666666664, "paired_cluster_bootstrap95_pp": [ -73.33333333333333, -20.0 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.5789473684210527, "jev_accuracy": 0.8947368421052632, "difference_pp": -31.57894736842105, "paired_cluster_bootstrap95_pp": [ -57.89473684210526, -5.2631578947368425 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 13, "jev_correct": 19, "local_accuracy": 0.65, "jev_accuracy": 0.95, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -50.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 13, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 17, "jev_correct": 20, "local_accuracy": 0.85, "jev_accuracy": 1.0, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -35.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 17, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.45, "jev_accuracy": 0.8, "difference_pp": -35.0, "paired_cluster_bootstrap95_pp": [ -55.0, -15.0 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 4, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 16, "jev_correct": 20, "local_accuracy": 0.7272727272727273, "jev_accuracy": 0.9090909090909091, "difference_pp": -18.181818181818183, "paired_cluster_bootstrap95_pp": [ -36.36363636363637, -4.545454545454546 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 16, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 412, "jev_correct": 629, "local_accuracy": 0.4577777777777778, "jev_accuracy": 0.6988888888888889, "difference_pp": -24.11111111111111, "paired_cluster_bootstrap95_pp": [ -29.21768329470041, -19.406848672955412 ], "local_only_correct": 98, "jev_only_correct": 315, "both_correct": 314, "both_wrong": 173, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 350, "jev_correct": 465, "local_accuracy": 0.6097560975609756, "jev_accuracy": 0.8101045296167247, "difference_pp": -20.034843205574912, "paired_cluster_bootstrap95_pp": [ -24.041811846689896, -16.027874564459932 ], "local_only_correct": 24, "jev_only_correct": 139, "both_correct": 326, "both_wrong": 85, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 221, "jev_correct": 490, "local_accuracy": 0.3157142857142857, "jev_accuracy": 0.7, "difference_pp": -38.42857142857143, "paired_cluster_bootstrap95_pp": [ -42.65134821806641, -33.94778153043309 ], "local_only_correct": 36, "jev_only_correct": 305, "both_correct": 185, "both_wrong": 174, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 18, "jev_correct": 97, "local_accuracy": 0.045, "jev_accuracy": 0.2425, "difference_pp": -19.75, "paired_cluster_bootstrap95_pp": [ -23.75, -15.75 ], "local_only_correct": 6, "jev_only_correct": 85, "both_correct": 12, "both_wrong": 297, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 22, "jev_correct": 28, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -35.714285714285715, -7.142857142857143 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 22, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 21, "jev_correct": 28, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -39.285714285714285, -10.714285714285714 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 21, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 36, "jev_correct": 40, "local_accuracy": 0.9, "jev_accuracy": 1.0, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -20.0, -2.5 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 36, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 37, "jev_correct": 40, "local_accuracy": 0.925, "jev_accuracy": 1.0, "difference_pp": -7.5, "paired_cluster_bootstrap95_pp": [ -17.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 37, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 31, "jev_correct": 39, "local_accuracy": 0.775, "jev_accuracy": 0.975, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -32.5, -7.5 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 31, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 32, "jev_correct": 39, "local_accuracy": 0.8, "jev_accuracy": 0.975, "difference_pp": -17.5, "paired_cluster_bootstrap95_pp": [ -30.0, -7.437500000000341 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 32, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 234, "jev_correct": 237, "local_accuracy": 0.585, "jev_accuracy": 0.5925, "difference_pp": -0.75, "paired_cluster_bootstrap95_pp": [ -4.75, 3.25 ], "local_only_correct": 32, "jev_only_correct": 35, "both_correct": 202, "both_wrong": 131, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 249, "jev_correct": 277, "local_accuracy": 0.83, "jev_accuracy": 0.9233333333333333, "difference_pp": -9.333333333333334, "paired_cluster_bootstrap95_pp": [ -13.333333333333334, -5.333333333333333 ], "local_only_correct": 5, "jev_only_correct": 33, "both_correct": 244, "both_wrong": 18, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 356, "jev_correct": 377, "local_accuracy": 0.89, "jev_accuracy": 0.9425, "difference_pp": -5.25, "paired_cluster_bootstrap95_pp": [ -9.0, -1.5 ], "local_only_correct": 16, "jev_only_correct": 37, "both_correct": 340, "both_wrong": 7, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 338, "jev_correct": 346, "local_accuracy": 0.845, "jev_accuracy": 0.865, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -4.5, 0.5 ], "local_only_correct": 9, "jev_only_correct": 17, "both_correct": 329, "both_wrong": 45, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 296, "jev_correct": 360, "local_accuracy": 0.74, "jev_accuracy": 0.9, "difference_pp": -16.0, "paired_cluster_bootstrap95_pp": [ -21.25, -10.5 ], "local_only_correct": 35, "jev_only_correct": 99, "both_correct": 261, "both_wrong": 5, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 186, "jev_correct": 225, "local_accuracy": 0.62, "jev_accuracy": 0.75, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -17.666666666666668, -8.333333333333334 ], "local_only_correct": 9, "jev_only_correct": 48, "both_correct": 177, "both_wrong": 66, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 315, "jev_correct": 393, "local_accuracy": 0.7875, "jev_accuracy": 0.9825, "difference_pp": -19.5, "paired_cluster_bootstrap95_pp": [ -23.25, -15.75 ], "local_only_correct": 0, "jev_only_correct": 78, "both_correct": 315, "both_wrong": 7, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 220, "jev_correct": 265, "local_accuracy": 0.55, "jev_accuracy": 0.6625, "difference_pp": -11.25, "paired_cluster_bootstrap95_pp": [ -16.25, -6.5 ], "local_only_correct": 29, "jev_only_correct": 74, "both_correct": 191, "both_wrong": 106, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 64, "jev_correct": 75, "local_accuracy": 0.64, "jev_accuracy": 0.75, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -19.0, -3.0 ], "local_only_correct": 4, "jev_only_correct": 15, "both_correct": 60, "both_wrong": 21, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 54, "jev_correct": 56, "local_accuracy": 0.54, "jev_accuracy": 0.56, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -11.0, 7.0 ], "local_only_correct": 10, "jev_only_correct": 12, "both_correct": 44, "both_wrong": 34, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 43, "jev_correct": 49, "local_accuracy": 0.43, "jev_accuracy": 0.49, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -14.0, 1.0 ], "local_only_correct": 4, "jev_only_correct": 10, "both_correct": 39, "both_wrong": 47, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 41, "jev_correct": 48, "local_accuracy": 0.41, "jev_accuracy": 0.48, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -17.0, 3.0 ], "local_only_correct": 11, "jev_only_correct": 18, "both_correct": 30, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 79, "jev_correct": 79, "local_accuracy": 0.79, "jev_accuracy": 0.79, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -5.0, 5.0 ], "local_only_correct": 4, "jev_only_correct": 4, "both_correct": 75, "both_wrong": 17, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 7, "jev_correct": 6, "local_accuracy": 0.07, "jev_accuracy": 0.06, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 3, "both_correct": 3, "both_wrong": 90, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 64, "jev_correct": 64, "local_accuracy": 0.64, "jev_accuracy": 0.64, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -8.0, 9.0 ], "local_only_correct": 9, "jev_only_correct": 9, "both_correct": 55, "both_wrong": 27, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 36, "jev_correct": 37, "local_accuracy": 0.36, "jev_accuracy": 0.37, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -11.0, 9.0 ], "local_only_correct": 11, "jev_only_correct": 12, "both_correct": 25, "both_wrong": 52, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 8, "jev_correct": 16, "local_accuracy": 0.08, "jev_accuracy": 0.16, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -16.0, -1.0 ], "local_only_correct": 3, "jev_only_correct": 11, "both_correct": 5, "both_wrong": 81, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 22, "jev_correct": 34, "local_accuracy": 0.22, "jev_accuracy": 0.34, "difference_pp": -12.0, "paired_cluster_bootstrap95_pp": [ -22.0, -3.0 ], "local_only_correct": 6, "jev_only_correct": 18, "both_correct": 16, "both_wrong": 60, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 79, "jev_correct": 83, "local_accuracy": 0.79, "jev_accuracy": 0.83, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -10.0, 1.0 ], "local_only_correct": 2, "jev_only_correct": 6, "both_correct": 77, "both_wrong": 15, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 14, "jev_correct": 21, "local_accuracy": 0.14, "jev_accuracy": 0.21, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -17.0, 2.0 ], "local_only_correct": 8, "jev_only_correct": 15, "both_correct": 6, "both_wrong": 71, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 47, "jev_correct": 51, "local_accuracy": 0.47, "jev_accuracy": 0.51, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -13.0, 5.0 ], "local_only_correct": 10, "jev_only_correct": 14, "both_correct": 37, "both_wrong": 39, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 61, "jev_correct": 65, "local_accuracy": 0.61, "jev_accuracy": 0.65, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -13.0, 6.0 ], "local_only_correct": 10, "jev_only_correct": 14, "both_correct": 51, "both_wrong": 25, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 43, "jev_correct": 39, "local_accuracy": 0.43, "jev_accuracy": 0.39, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -4.0, 12.0 ], "local_only_correct": 10, "jev_only_correct": 6, "both_correct": 33, "both_wrong": 51, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 9, "jev_correct": 9, "local_accuracy": 0.09, "jev_accuracy": 0.09, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -7.0, 7.0 ], "local_only_correct": 6, "jev_only_correct": 6, "both_correct": 3, "both_wrong": 85, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 60, "jev_correct": 59, "local_accuracy": 0.6, "jev_accuracy": 0.59, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 8.0 ], "local_only_correct": 6, "jev_only_correct": 5, "both_correct": 54, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 86, "jev_correct": 91, "local_accuracy": 0.86, "jev_accuracy": 0.91, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.0, 2.0 ], "local_only_correct": 4, "jev_only_correct": 9, "both_correct": 82, "both_wrong": 5, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 60, "jev_correct": 88, "local_accuracy": 0.6, "jev_accuracy": 0.88, "difference_pp": -28.000000000000004, "paired_cluster_bootstrap95_pp": [ -38.0, -18.0 ], "local_only_correct": 2, "jev_only_correct": 30, "both_correct": 58, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 50, "jev_correct": 48, "local_accuracy": 0.5, "jev_accuracy": 0.48, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -7.0, 11.0 ], "local_only_correct": 12, "jev_only_correct": 10, "both_correct": 38, "both_wrong": 40, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2089, "jev_correct": 2550, "local_accuracy": 0.7086160108548168, "jev_accuracy": 0.8649932157394844, "difference_pp": -15.637720488466758, "paired_cluster_bootstrap95_pp": [ -17.19810040705563, -14.110413839891455 ], "local_only_correct": 69, "jev_only_correct": 530, "both_correct": 2020, "both_wrong": 329, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 1682, "jev_correct": 2412, "local_accuracy": 0.5705563093622795, "jev_accuracy": 0.8181818181818182, "difference_pp": -24.762550881953867, "paired_cluster_bootstrap95_pp": [ -26.526458616010856, -22.99864314789688 ], "local_only_correct": 81, "jev_only_correct": 811, "both_correct": 1601, "both_wrong": 455, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 612, "jev_correct": 843, "local_accuracy": 0.5217391304347826, "jev_accuracy": 0.7186700767263428, "difference_pp": -19.69309462915601, "paired_cluster_bootstrap95_pp": [ -23.0975814073374, -16.507982737165403 ], "local_only_correct": 77, "jev_only_correct": 308, "both_correct": 535, "both_wrong": 253, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "jevk5": { "fresh/news": { "n": 200, "local_correct": 173, "jev_correct": 178, "local_accuracy": 0.865, "jev_accuracy": 0.89, "difference_pp": -2.5, "paired_cluster_bootstrap95_pp": [ -6.0, 0.5 ], "local_only_correct": 3, "jev_only_correct": 8, "both_correct": 170, "both_wrong": 19, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 121, "jev_correct": 120, "local_accuracy": 0.605, "jev_accuracy": 0.6, "difference_pp": 0.5, "paired_cluster_bootstrap95_pp": [ -3.5, 5.0 ], "local_only_correct": 10, "jev_only_correct": 9, "both_correct": 111, "both_wrong": 70, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 166, "jev_correct": 196, "local_accuracy": 0.83, "jev_accuracy": 0.98, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -20.5, -10.0 ], "local_only_correct": 0, "jev_only_correct": 30, "both_correct": 166, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 180, "jev_correct": 179, "local_accuracy": 0.9, "jev_accuracy": 0.895, "difference_pp": 0.5, "paired_cluster_bootstrap95_pp": [ -3.0, 4.012499999999932 ], "local_only_correct": 7, "jev_only_correct": 6, "both_correct": 173, "both_wrong": 14, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 10, "jev_correct": 11, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9166666666666666, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 10, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 9, "jev_correct": 10, "local_accuracy": 0.75, "jev_accuracy": 0.8333333333333334, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 9, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 86, "jev_correct": 91, "local_accuracy": 0.86, "jev_accuracy": 0.91, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -10.0, -1.0 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 86, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 49, "jev_correct": 48, "local_accuracy": 0.49, "jev_accuracy": 0.48, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 10.0 ], "local_only_correct": 9, "jev_only_correct": 8, "both_correct": 40, "both_wrong": 43, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 76, "jev_correct": 89, "local_accuracy": 0.76, "jev_accuracy": 0.89, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -21.0, -5.975000000000136 ], "local_only_correct": 2, "jev_only_correct": 15, "both_correct": 74, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 86, "jev_correct": 81, "local_accuracy": 0.7747747747747747, "jev_accuracy": 0.7297297297297297, "difference_pp": 4.504504504504505, "paired_cluster_bootstrap95_pp": [ -2.7027027027027026, 11.711711711711711 ], "local_only_correct": 11, "jev_only_correct": 6, "both_correct": 75, "both_wrong": 19, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 68, "jev_correct": 70, "local_accuracy": 0.9444444444444444, "jev_accuracy": 0.9722222222222222, "difference_pp": -2.7777777777777777, "paired_cluster_bootstrap95_pp": [ -11.11111111111111, 4.166666666666667 ], "local_only_correct": 2, "jev_only_correct": 4, "both_correct": 66, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.8947368421052632, "jev_accuracy": 0.9473684210526315, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -26.31578947368421, 10.526315789473685 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9090909090909091, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.7647058823529411, "jev_accuracy": 0.9411764705882353, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8125, "jev_accuracy": 0.9375, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 19, "jev_correct": 21, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9545454545454546, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -22.727272727272727, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9444444444444444, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -22.22222222222222, 11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 9, "jev_correct": 15, "local_accuracy": 0.5625, "jev_accuracy": 0.9375, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 9, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9444444444444444, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.85, "jev_accuracy": 0.9, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -20.0, 10.0 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 12, "jev_correct": 18, "local_accuracy": 0.5714285714285714, "jev_accuracy": 0.8571428571428571, "difference_pp": -28.57142857142857, "paired_cluster_bootstrap95_pp": [ -52.38095238095238, -4.761904761904762 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 11, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 15, "jev_correct": 14, "local_accuracy": 0.9375, "jev_accuracy": 0.875, "difference_pp": 6.25, "paired_cluster_bootstrap95_pp": [ 0.0, 18.75 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.6875, "jev_accuracy": 0.875, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -43.75, 6.25 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -41.1764705882353, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.8125, "jev_accuracy": 1.0, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -43.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6111111111111112, "jev_accuracy": 0.9444444444444444, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.75, "jev_accuracy": 0.9375, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.6842105263157895, "jev_accuracy": 0.8947368421052632, "difference_pp": -21.052631578947366, "paired_cluster_bootstrap95_pp": [ -42.10526315789474, -5.2631578947368425 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 2, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 14, "jev_correct": 19, "local_accuracy": 0.7, "jev_accuracy": 0.95, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -45.0, -5.0 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 14, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 16, "jev_correct": 16, "local_accuracy": 0.8, "jev_accuracy": 0.8, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -15.0, 15.0 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 3, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9090909090909091, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 530, "jev_correct": 629, "local_accuracy": 0.5888888888888889, "jev_accuracy": 0.6988888888888889, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -14.447501059514439, -7.9491218981639 ], "local_only_correct": 44, "jev_only_correct": 143, "both_correct": 486, "both_wrong": 227, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 417, "jev_correct": 465, "local_accuracy": 0.7264808362369338, "jev_accuracy": 0.8101045296167247, "difference_pp": -8.362369337979095, "paired_cluster_bootstrap95_pp": [ -11.846689895470384, -5.052264808362369 ], "local_only_correct": 31, "jev_only_correct": 79, "both_correct": 386, "both_wrong": 78, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 333, "jev_correct": 490, "local_accuracy": 0.4757142857142857, "jev_accuracy": 0.7, "difference_pp": -22.428571428571427, "paired_cluster_bootstrap95_pp": [ -26.453602087399517, -18.31390150380943 ], "local_only_correct": 43, "jev_only_correct": 200, "both_correct": 290, "both_wrong": 167, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 55, "jev_correct": 97, "local_accuracy": 0.1375, "jev_accuracy": 0.2425, "difference_pp": -10.5, "paired_cluster_bootstrap95_pp": [ -15.00625, -5.75 ], "local_only_correct": 29, "jev_only_correct": 71, "both_correct": 26, "both_wrong": 274, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 35, "jev_correct": 40, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -22.5, -2.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 35, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 37, "jev_correct": 40, "local_accuracy": 0.925, "jev_accuracy": 1.0, "difference_pp": -7.5, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 37, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 32, "jev_correct": 39, "local_accuracy": 0.8, "jev_accuracy": 0.975, "difference_pp": -17.5, "paired_cluster_bootstrap95_pp": [ -30.0, -7.5 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 32, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 36, "jev_correct": 39, "local_accuracy": 0.9, "jev_accuracy": 0.975, "difference_pp": -7.5, "paired_cluster_bootstrap95_pp": [ -17.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 36, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 240, "jev_correct": 237, "local_accuracy": 0.6, "jev_accuracy": 0.5925, "difference_pp": 0.75, "paired_cluster_bootstrap95_pp": [ -2.5, 4.25 ], "local_only_correct": 26, "jev_only_correct": 23, "both_correct": 214, "both_wrong": 137, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 273, "jev_correct": 277, "local_accuracy": 0.91, "jev_accuracy": 0.9233333333333333, "difference_pp": -1.3333333333333335, "paired_cluster_bootstrap95_pp": [ -4.333333333333333, 2.0 ], "local_only_correct": 9, "jev_only_correct": 13, "both_correct": 264, "both_wrong": 14, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 356, "jev_correct": 377, "local_accuracy": 0.89, "jev_accuracy": 0.9425, "difference_pp": -5.25, "paired_cluster_bootstrap95_pp": [ -8.25, -2.5 ], "local_only_correct": 7, "jev_only_correct": 28, "both_correct": 349, "both_wrong": 16, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 333, "jev_correct": 346, "local_accuracy": 0.8325, "jev_accuracy": 0.865, "difference_pp": -3.25, "paired_cluster_bootstrap95_pp": [ -5.75, -0.9937500000000341 ], "local_only_correct": 6, "jev_only_correct": 19, "both_correct": 327, "both_wrong": 48, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 354, "jev_correct": 360, "local_accuracy": 0.885, "jev_accuracy": 0.9, "difference_pp": -1.5, "paired_cluster_bootstrap95_pp": [ -3.75, 0.75 ], "local_only_correct": 8, "jev_only_correct": 14, "both_correct": 346, "both_wrong": 32, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 229, "jev_correct": 225, "local_accuracy": 0.7633333333333333, "jev_accuracy": 0.75, "difference_pp": 1.3333333333333335, "paired_cluster_bootstrap95_pp": [ -1.6666666666666667, 4.333333333333333 ], "local_only_correct": 13, "jev_only_correct": 9, "both_correct": 216, "both_wrong": 62, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 337, "jev_correct": 393, "local_accuracy": 0.8425, "jev_accuracy": 0.9825, "difference_pp": -14.000000000000002, "paired_cluster_bootstrap95_pp": [ -17.5, -10.75 ], "local_only_correct": 1, "jev_only_correct": 57, "both_correct": 336, "both_wrong": 6, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 264, "jev_correct": 265, "local_accuracy": 0.66, "jev_accuracy": 0.6625, "difference_pp": -0.25, "paired_cluster_bootstrap95_pp": [ -4.0, 3.5 ], "local_only_correct": 29, "jev_only_correct": 30, "both_correct": 235, "both_wrong": 106, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 70, "jev_correct": 75, "local_accuracy": 0.7, "jev_accuracy": 0.75, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -11.0, 0.0 ], "local_only_correct": 2, "jev_only_correct": 7, "both_correct": 68, "both_wrong": 23, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 59, "jev_correct": 56, "local_accuracy": 0.59, "jev_accuracy": 0.56, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -3.0249999999999986, 10.0 ], "local_only_correct": 8, "jev_only_correct": 5, "both_correct": 51, "both_wrong": 36, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 49, "jev_correct": 49, "local_accuracy": 0.49, "jev_accuracy": 0.49, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -5.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 4, "both_correct": 45, "both_wrong": 47, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 46, "jev_correct": 48, "local_accuracy": 0.46, "jev_accuracy": 0.48, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -12.0, 7.0 ], "local_only_correct": 11, "jev_only_correct": 13, "both_correct": 35, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 80, "jev_correct": 79, "local_accuracy": 0.8, "jev_accuracy": 0.79, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 3, "both_correct": 76, "both_wrong": 17, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 10, "jev_correct": 6, "local_accuracy": 0.1, "jev_accuracy": 0.06, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -1.0, 9.0 ], "local_only_correct": 6, "jev_only_correct": 2, "both_correct": 4, "both_wrong": 88, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 63, "jev_correct": 64, "local_accuracy": 0.63, "jev_accuracy": 0.64, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -9.0, 7.0 ], "local_only_correct": 8, "jev_only_correct": 9, "both_correct": 55, "both_wrong": 28, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 37, "jev_correct": 37, "local_accuracy": 0.37, "jev_accuracy": 0.37, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -6.0, 6.0 ], "local_only_correct": 5, "jev_only_correct": 5, "both_correct": 32, "both_wrong": 58, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.12, "jev_accuracy": 0.16, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -12.0, 4.0 ], "local_only_correct": 6, "jev_only_correct": 10, "both_correct": 6, "both_wrong": 78, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 33, "jev_correct": 34, "local_accuracy": 0.33, "jev_accuracy": 0.34, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -9.0, 6.0 ], "local_only_correct": 7, "jev_only_correct": 8, "both_correct": 26, "both_wrong": 59, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 86, "jev_correct": 83, "local_accuracy": 0.86, "jev_accuracy": 0.83, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -1.0, 8.0 ], "local_only_correct": 4, "jev_only_correct": 1, "both_correct": 82, "both_wrong": 13, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 14, "jev_correct": 21, "local_accuracy": 0.14, "jev_accuracy": 0.21, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -15.0, 1.0 ], "local_only_correct": 5, "jev_only_correct": 12, "both_correct": 9, "both_wrong": 74, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 49, "jev_correct": 51, "local_accuracy": 0.49, "jev_accuracy": 0.51, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -9.0, 5.0 ], "local_only_correct": 5, "jev_only_correct": 7, "both_correct": 44, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 68, "jev_correct": 65, "local_accuracy": 0.68, "jev_accuracy": 0.65, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -1.0, 7.0 ], "local_only_correct": 4, "jev_only_correct": 1, "both_correct": 64, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 35, "jev_correct": 39, "local_accuracy": 0.35, "jev_accuracy": 0.39, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -11.0, 3.0 ], "local_only_correct": 5, "jev_only_correct": 9, "both_correct": 30, "both_wrong": 56, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 12, "jev_correct": 9, "local_accuracy": 0.12, "jev_accuracy": 0.09, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -3.0, 9.0 ], "local_only_correct": 7, "jev_only_correct": 4, "both_correct": 5, "both_wrong": 84, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 58, "jev_correct": 59, "local_accuracy": 0.58, "jev_accuracy": 0.59, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 5.0 ], "local_only_correct": 4, "jev_only_correct": 5, "both_correct": 54, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 88, "jev_correct": 91, "local_accuracy": 0.88, "jev_accuracy": 0.91, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -7.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 88, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 77, "jev_correct": 88, "local_accuracy": 0.77, "jev_accuracy": 0.88, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -18.0, -4.0 ], "local_only_correct": 1, "jev_only_correct": 12, "both_correct": 76, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 50, "jev_correct": 48, "local_accuracy": 0.5, "jev_accuracy": 0.48, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -6.0, 11.0 ], "local_only_correct": 10, "jev_only_correct": 8, "both_correct": 40, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2375, "jev_correct": 2550, "local_accuracy": 0.8056309362279511, "jev_accuracy": 0.8649932157394844, "difference_pp": -5.936227951153325, "paired_cluster_bootstrap95_pp": [ -7.123473541383989, -4.715061058344641 ], "local_only_correct": 71, "jev_only_correct": 246, "both_correct": 2304, "both_wrong": 327, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2133, "jev_correct": 2412, "local_accuracy": 0.7235413839891451, "jev_accuracy": 0.8181818181818182, "difference_pp": -9.4640434192673, "paired_cluster_bootstrap95_pp": [ -10.753900949796472, -8.14111261872456 ], "local_only_correct": 70, "jev_only_correct": 349, "both_correct": 2063, "both_wrong": 466, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 813, "jev_correct": 843, "local_accuracy": 0.6930946291560103, "jev_accuracy": 0.7186700767263428, "difference_pp": -2.557544757033248, "paired_cluster_bootstrap95_pp": [ -5.0580853637644525, -0.19989715964740506 ], "local_only_correct": 96, "jev_only_correct": 126, "both_correct": 717, "both_wrong": 234, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "julia": { "fresh/news": { "n": 200, "local_correct": 179, "jev_correct": 178, "local_accuracy": 0.895, "jev_accuracy": 0.89, "difference_pp": 0.5, "paired_cluster_bootstrap95_pp": [ -4.5, 6.0 ], "local_only_correct": 14, "jev_only_correct": 13, "both_correct": 165, "both_wrong": 8, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 182, "jev_correct": 120, "local_accuracy": 0.91, "jev_accuracy": 0.6, "difference_pp": 31.0, "paired_cluster_bootstrap95_pp": [ 24.5, 38.5 ], "local_only_correct": 65, "jev_only_correct": 3, "both_correct": 117, "both_wrong": 15, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 93, "jev_correct": 196, "local_accuracy": 0.465, "jev_accuracy": 0.98, "difference_pp": -51.5, "paired_cluster_bootstrap95_pp": [ -59.0, -44.5 ], "local_only_correct": 3, "jev_only_correct": 106, "both_correct": 90, "both_wrong": 1, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 76, "jev_correct": 179, "local_accuracy": 0.38, "jev_accuracy": 0.895, "difference_pp": -51.5, "paired_cluster_bootstrap95_pp": [ -60.5125, -42.0 ], "local_only_correct": 20, "jev_only_correct": 123, "both_correct": 56, "both_wrong": 1, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 3, "jev_correct": 11, "local_accuracy": 0.25, "jev_accuracy": 0.9166666666666666, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -91.66666666666667, -41.666666666666664 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 3, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 4, "jev_correct": 10, "local_accuracy": 0.3333333333333333, "jev_accuracy": 0.8333333333333334, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 4, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 6, "jev_correct": 12, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 6, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 5, "jev_correct": 12, "local_accuracy": 0.4166666666666667, "jev_accuracy": 1.0, "difference_pp": -58.333333333333336, "paired_cluster_bootstrap95_pp": [ -83.33333333333333, -25.0 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 5, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 94, "jev_correct": 91, "local_accuracy": 0.94, "jev_accuracy": 0.91, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -4.0, 10.0 ], "local_only_correct": 8, "jev_only_correct": 5, "both_correct": 86, "both_wrong": 1, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 86, "jev_correct": 48, "local_accuracy": 0.86, "jev_accuracy": 0.48, "difference_pp": 38.0, "paired_cluster_bootstrap95_pp": [ 28.0, 49.0 ], "local_only_correct": 41, "jev_only_correct": 3, "both_correct": 45, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 0, "jev_correct": 89, "local_accuracy": 0.0, "jev_accuracy": 0.89, "difference_pp": -89.0, "paired_cluster_bootstrap95_pp": [ -95.0, -82.0 ], "local_only_correct": 0, "jev_only_correct": 89, "both_correct": 0, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 35, "jev_correct": 48, "local_accuracy": 0.7291666666666666, "jev_accuracy": 1.0, "difference_pp": -27.083333333333332, "paired_cluster_bootstrap95_pp": [ -39.583333333333336, -14.583333333333334 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 35, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/hard": { "n": 111, "local_correct": 40, "jev_correct": 81, "local_accuracy": 0.36036036036036034, "jev_accuracy": 0.7297297297297297, "difference_pp": -36.93693693693694, "paired_cluster_bootstrap95_pp": [ -48.648648648648646, -25.225225225225227 ], "local_only_correct": 9, "jev_only_correct": 50, "both_correct": 31, "both_wrong": 21, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 34, "jev_correct": 70, "local_accuracy": 0.4722222222222222, "jev_accuracy": 0.9722222222222222, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -36.076388888889085 ], "local_only_correct": 2, "jev_only_correct": 38, "both_correct": 32, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 12, "jev_correct": 18, "local_accuracy": 0.631578947368421, "jev_accuracy": 0.9473684210526315, "difference_pp": -31.57894736842105, "paired_cluster_bootstrap95_pp": [ -57.89473684210526, -5.2631578947368425 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 11, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 8, "jev_correct": 16, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 8, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 11, "jev_correct": 20, "local_accuracy": 0.5, "jev_accuracy": 0.9090909090909091, "difference_pp": -40.909090909090914, "paired_cluster_bootstrap95_pp": [ -68.18181818181819, -13.636363636363637 ], "local_only_correct": 2, "jev_only_correct": 11, "both_correct": 9, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 11, "jev_correct": 16, "local_accuracy": 0.6875, "jev_accuracy": 1.0, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 11, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 7, "jev_correct": 18, "local_accuracy": 0.3888888888888889, "jev_accuracy": 1.0, "difference_pp": -61.111111111111114, "paired_cluster_bootstrap95_pp": [ -83.33333333333333, -38.888888888888886 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 7, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 11, "jev_correct": 19, "local_accuracy": 0.5789473684210527, "jev_accuracy": 1.0, "difference_pp": -42.10526315789473, "paired_cluster_bootstrap95_pp": [ -63.1578947368421, -21.05263157894737 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 11, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 5, "jev_correct": 16, "local_accuracy": 0.29411764705882354, "jev_accuracy": 0.9411764705882353, "difference_pp": -64.70588235294117, "paired_cluster_bootstrap95_pp": [ -88.23529411764706, -41.1764705882353 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 5, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4666666666666667, "jev_accuracy": 1.0, "difference_pp": -53.333333333333336, "paired_cluster_bootstrap95_pp": [ -80.0, -26.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 9, "jev_correct": 15, "local_accuracy": 0.5625, "jev_accuracy": 0.9375, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -68.75, -6.25 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 5, "jev_correct": 18, "local_accuracy": 0.2777777777777778, "jev_accuracy": 1.0, "difference_pp": -72.22222222222221, "paired_cluster_bootstrap95_pp": [ -88.88888888888889, -50.0 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 5, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 9, "jev_correct": 21, "local_accuracy": 0.4090909090909091, "jev_accuracy": 0.9545454545454546, "difference_pp": -54.54545454545454, "paired_cluster_bootstrap95_pp": [ -72.72727272727273, -31.818181818181817 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 9, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 12, "jev_correct": 19, "local_accuracy": 0.631578947368421, "jev_accuracy": 1.0, "difference_pp": -36.84210526315789, "paired_cluster_bootstrap95_pp": [ -57.89473684210526, -15.789473684210526 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 12, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4666666666666667, "jev_accuracy": 1.0, "difference_pp": -53.333333333333336, "paired_cluster_bootstrap95_pp": [ -80.0, -26.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4666666666666667, "jev_accuracy": 1.0, "difference_pp": -53.333333333333336, "paired_cluster_bootstrap95_pp": [ -80.0, -26.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 12, "jev_correct": 18, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 12, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.4444444444444444, "jev_accuracy": 0.9444444444444444, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -77.77777777777777, -22.22222222222222 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 7, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 3, "jev_correct": 15, "local_accuracy": 0.1875, "jev_accuracy": 0.9375, "difference_pp": -75.0, "paired_cluster_bootstrap95_pp": [ -93.75, -50.0 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 3, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.4444444444444444, "jev_accuracy": 0.9444444444444444, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -72.22222222222223, -27.77777777777778 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 8, "jev_correct": 18, "local_accuracy": 0.4, "jev_accuracy": 0.9, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 1, "jev_only_correct": 11, "both_correct": 7, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 4, "jev_correct": 18, "local_accuracy": 0.19047619047619047, "jev_accuracy": 0.8571428571428571, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -90.47619047619048, -42.857142857142854 ], "local_only_correct": 1, "jev_only_correct": 15, "both_correct": 3, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 8, "jev_correct": 16, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 8, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 13, "jev_correct": 23, "local_accuracy": 0.5652173913043478, "jev_accuracy": 1.0, "difference_pp": -43.47826086956522, "paired_cluster_bootstrap95_pp": [ -65.21739130434783, -26.08695652173913 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 13, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 4, "jev_correct": 14, "local_accuracy": 0.2857142857142857, "jev_accuracy": 1.0, "difference_pp": -71.42857142857143, "paired_cluster_bootstrap95_pp": [ -92.85714285714286, -50.0 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 4, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 4, "jev_correct": 14, "local_accuracy": 0.25, "jev_accuracy": 0.875, "difference_pp": -62.5, "paired_cluster_bootstrap95_pp": [ -87.5, -37.5 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 4, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 5, "jev_correct": 14, "local_accuracy": 0.3125, "jev_accuracy": 0.875, "difference_pp": -56.25, "paired_cluster_bootstrap95_pp": [ -81.25, -25.0 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 4, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 5, "jev_correct": 17, "local_accuracy": 0.29411764705882354, "jev_accuracy": 1.0, "difference_pp": -70.58823529411765, "paired_cluster_bootstrap95_pp": [ -88.23529411764706, -47.05882352941177 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 5, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 7, "jev_correct": 16, "local_accuracy": 0.4375, "jev_accuracy": 1.0, "difference_pp": -56.25, "paired_cluster_bootstrap95_pp": [ -81.25, -31.25 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 7, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 5, "jev_correct": 16, "local_accuracy": 0.3125, "jev_accuracy": 1.0, "difference_pp": -68.75, "paired_cluster_bootstrap95_pp": [ -87.5, -43.75 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 5, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 7, "jev_correct": 18, "local_accuracy": 0.3888888888888889, "jev_accuracy": 1.0, "difference_pp": -61.111111111111114, "paired_cluster_bootstrap95_pp": [ -83.33333333333333, -38.888888888888886 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 7, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 7, "jev_correct": 17, "local_accuracy": 0.4117647058823529, "jev_accuracy": 1.0, "difference_pp": -58.82352941176471, "paired_cluster_bootstrap95_pp": [ -82.3529411764706, -35.294117647058826 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 7, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 4, "jev_correct": 18, "local_accuracy": 0.2222222222222222, "jev_accuracy": 1.0, "difference_pp": -77.77777777777779, "paired_cluster_bootstrap95_pp": [ -94.44444444444444, -55.55555555555556 ], "local_only_correct": 0, "jev_only_correct": 14, "both_correct": 4, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 6, "jev_correct": 17, "local_accuracy": 0.3333333333333333, "jev_accuracy": 0.9444444444444444, "difference_pp": -61.111111111111114, "paired_cluster_bootstrap95_pp": [ -83.33333333333333, -38.888888888888886 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 6, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5333333333333333, "jev_accuracy": 1.0, "difference_pp": -46.666666666666664, "paired_cluster_bootstrap95_pp": [ -73.33333333333333, -20.0 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 7, "jev_correct": 17, "local_accuracy": 0.3684210526315789, "jev_accuracy": 0.8947368421052632, "difference_pp": -52.63157894736842, "paired_cluster_bootstrap95_pp": [ -78.94736842105263, -26.31578947368421 ], "local_only_correct": 1, "jev_only_correct": 11, "both_correct": 6, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 8, "jev_correct": 18, "local_accuracy": 0.4444444444444444, "jev_accuracy": 1.0, "difference_pp": -55.55555555555556, "paired_cluster_bootstrap95_pp": [ -77.77777777777777, -33.333333333333336 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 8, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 6, "jev_correct": 19, "local_accuracy": 0.3, "jev_accuracy": 0.95, "difference_pp": -65.0, "paired_cluster_bootstrap95_pp": [ -85.0, -45.0 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 6, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 9, "jev_correct": 20, "local_accuracy": 0.45, "jev_accuracy": 1.0, "difference_pp": -55.00000000000001, "paired_cluster_bootstrap95_pp": [ -75.0, -35.0 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 9, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 10, "jev_correct": 16, "local_accuracy": 0.5, "jev_accuracy": 0.8, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -60.0, 5.0 ], "local_only_correct": 3, "jev_only_correct": 9, "both_correct": 7, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 11, "jev_correct": 20, "local_accuracy": 0.5, "jev_accuracy": 0.9090909090909091, "difference_pp": -40.909090909090914, "paired_cluster_bootstrap95_pp": [ -63.63636363636363, -13.636363636363637 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 10, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 326, "jev_correct": 629, "local_accuracy": 0.3622222222222222, "jev_accuracy": 0.6988888888888889, "difference_pp": -33.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.50843054485991, -26.741707394582274 ], "local_only_correct": 66, "jev_only_correct": 369, "both_correct": 260, "both_wrong": 205, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 195, "jev_correct": 465, "local_accuracy": 0.3397212543554007, "jev_accuracy": 0.8101045296167247, "difference_pp": -47.038327526132406, "paired_cluster_bootstrap95_pp": [ -51.56794425087108, -42.508710801393725 ], "local_only_correct": 12, "jev_only_correct": 282, "both_correct": 183, "both_wrong": 97, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 159, "jev_correct": 490, "local_accuracy": 0.22714285714285715, "jev_accuracy": 0.7, "difference_pp": -47.285714285714285, "paired_cluster_bootstrap95_pp": [ -51.846854707792204, -42.650474019249764 ], "local_only_correct": 45, "jev_only_correct": 376, "both_correct": 114, "both_wrong": 165, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 107, "jev_correct": 97, "local_accuracy": 0.2675, "jev_accuracy": 0.2425, "difference_pp": 2.5, "paired_cluster_bootstrap95_pp": [ -3.0, 8.0 ], "local_only_correct": 67, "jev_only_correct": 57, "both_correct": 40, "both_wrong": 236, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 18, "jev_correct": 28, "local_accuracy": 0.6428571428571429, "jev_accuracy": 1.0, "difference_pp": -35.714285714285715, "paired_cluster_bootstrap95_pp": [ -53.57142857142857, -21.428571428571427 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 18, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 21, "jev_correct": 28, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, -10.714285714285714 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 21, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 14, "jev_correct": 28, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -67.85714285714286, -32.142857142857146 ], "local_only_correct": 0, "jev_only_correct": 14, "both_correct": 14, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 19, "jev_correct": 28, "local_accuracy": 0.6785714285714286, "jev_accuracy": 1.0, "difference_pp": -32.142857142857146, "paired_cluster_bootstrap95_pp": [ -50.0, -14.285714285714286 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 19, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 24, "jev_correct": 40, "local_accuracy": 0.6, "jev_accuracy": 1.0, "difference_pp": -40.0, "paired_cluster_bootstrap95_pp": [ -55.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 16, "both_correct": 24, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 19, "jev_correct": 40, "local_accuracy": 0.475, "jev_accuracy": 1.0, "difference_pp": -52.5, "paired_cluster_bootstrap95_pp": [ -67.5, -37.5 ], "local_only_correct": 0, "jev_only_correct": 21, "both_correct": 19, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 17, "jev_correct": 39, "local_accuracy": 0.425, "jev_accuracy": 0.975, "difference_pp": -55.00000000000001, "paired_cluster_bootstrap95_pp": [ -70.0625, -37.5 ], "local_only_correct": 1, "jev_only_correct": 23, "both_correct": 16, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 11, "jev_correct": 39, "local_accuracy": 0.275, "jev_accuracy": 0.975, "difference_pp": -70.0, "paired_cluster_bootstrap95_pp": [ -85.0, -52.5 ], "local_only_correct": 1, "jev_only_correct": 29, "both_correct": 10, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 376, "jev_correct": 237, "local_accuracy": 0.94, "jev_accuracy": 0.5925, "difference_pp": 34.75, "paired_cluster_bootstrap95_pp": [ 30.0, 39.506249999999966 ], "local_only_correct": 143, "jev_only_correct": 4, "both_correct": 233, "both_wrong": 20, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 198, "jev_correct": 277, "local_accuracy": 0.66, "jev_accuracy": 0.9233333333333333, "difference_pp": -26.333333333333332, "paired_cluster_bootstrap95_pp": [ -31.666666666666668, -20.666666666666668 ], "local_only_correct": 7, "jev_only_correct": 86, "both_correct": 191, "both_wrong": 16, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 301, "jev_correct": 377, "local_accuracy": 0.7525, "jev_accuracy": 0.9425, "difference_pp": -19.0, "paired_cluster_bootstrap95_pp": [ -23.75, -14.5 ], "local_only_correct": 14, "jev_only_correct": 90, "both_correct": 287, "both_wrong": 9, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 344, "jev_correct": 346, "local_accuracy": 0.86, "jev_accuracy": 0.865, "difference_pp": -0.5, "paired_cluster_bootstrap95_pp": [ -4.0, 2.75 ], "local_only_correct": 23, "jev_only_correct": 25, "both_correct": 321, "both_wrong": 31, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 150, "jev_correct": 360, "local_accuracy": 0.375, "jev_accuracy": 0.9, "difference_pp": -52.5, "paired_cluster_bootstrap95_pp": [ -58.75, -46.243750000000034 ], "local_only_correct": 35, "jev_only_correct": 245, "both_correct": 115, "both_wrong": 5, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 156, "jev_correct": 225, "local_accuracy": 0.52, "jev_accuracy": 0.75, "difference_pp": -23.0, "paired_cluster_bootstrap95_pp": [ -29.0, -17.0 ], "local_only_correct": 16, "jev_only_correct": 85, "both_correct": 140, "both_wrong": 59, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 180, "jev_correct": 393, "local_accuracy": 0.45, "jev_accuracy": 0.9825, "difference_pp": -53.25, "paired_cluster_bootstrap95_pp": [ -58.25, -48.25 ], "local_only_correct": 5, "jev_only_correct": 218, "both_correct": 175, "both_wrong": 2, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 215, "jev_correct": 265, "local_accuracy": 0.5375, "jev_accuracy": 0.6625, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -19.0, -6.25 ], "local_only_correct": 64, "jev_only_correct": 114, "both_correct": 151, "both_wrong": 71, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 49, "jev_correct": 75, "local_accuracy": 0.49, "jev_accuracy": 0.75, "difference_pp": -26.0, "paired_cluster_bootstrap95_pp": [ -40.0, -12.0 ], "local_only_correct": 17, "jev_only_correct": 43, "both_correct": 32, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 43, "jev_correct": 56, "local_accuracy": 0.43, "jev_accuracy": 0.56, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -22.0, -4.0 ], "local_only_correct": 6, "jev_only_correct": 19, "both_correct": 37, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 36, "jev_correct": 49, "local_accuracy": 0.36, "jev_accuracy": 0.49, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -28.025, 3.0 ], "local_only_correct": 25, "jev_only_correct": 38, "both_correct": 11, "both_wrong": 26, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 20, "jev_correct": 48, "local_accuracy": 0.2, "jev_accuracy": 0.48, "difference_pp": -28.000000000000004, "paired_cluster_bootstrap95_pp": [ -38.0, -18.975000000000136 ], "local_only_correct": 2, "jev_only_correct": 30, "both_correct": 18, "both_wrong": 50, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 66, "jev_correct": 79, "local_accuracy": 0.66, "jev_accuracy": 0.79, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -22.0, -4.0 ], "local_only_correct": 6, "jev_only_correct": 19, "both_correct": 60, "both_wrong": 15, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 4, "jev_correct": 6, "local_accuracy": 0.04, "jev_accuracy": 0.06, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -8.0, 4.0 ], "local_only_correct": 4, "jev_only_correct": 6, "both_correct": 0, "both_wrong": 90, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 47, "jev_correct": 64, "local_accuracy": 0.47, "jev_accuracy": 0.64, "difference_pp": -17.0, "paired_cluster_bootstrap95_pp": [ -27.0, -7.0 ], "local_only_correct": 8, "jev_only_correct": 25, "both_correct": 39, "both_wrong": 28, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 34, "jev_correct": 37, "local_accuracy": 0.34, "jev_accuracy": 0.37, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -14.0, 8.0 ], "local_only_correct": 15, "jev_only_correct": 18, "both_correct": 19, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 0, "jev_correct": 16, "local_accuracy": 0.0, "jev_accuracy": 0.16, "difference_pp": -16.0, "paired_cluster_bootstrap95_pp": [ -24.0, -9.0 ], "local_only_correct": 0, "jev_only_correct": 16, "both_correct": 0, "both_wrong": 84, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 0, "jev_correct": 34, "local_accuracy": 0.0, "jev_accuracy": 0.34, "difference_pp": -34.0, "paired_cluster_bootstrap95_pp": [ -43.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 34, "both_correct": 0, "both_wrong": 66, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 54, "jev_correct": 83, "local_accuracy": 0.54, "jev_accuracy": 0.83, "difference_pp": -28.999999999999996, "paired_cluster_bootstrap95_pp": [ -40.0, -18.0 ], "local_only_correct": 5, "jev_only_correct": 34, "both_correct": 49, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 0, "jev_correct": 21, "local_accuracy": 0.0, "jev_accuracy": 0.21, "difference_pp": -21.0, "paired_cluster_bootstrap95_pp": [ -29.025, -13.0 ], "local_only_correct": 0, "jev_only_correct": 21, "both_correct": 0, "both_wrong": 79, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 10, "jev_correct": 51, "local_accuracy": 0.1, "jev_accuracy": 0.51, "difference_pp": -41.0, "paired_cluster_bootstrap95_pp": [ -52.0, -30.0 ], "local_only_correct": 3, "jev_only_correct": 44, "both_correct": 7, "both_wrong": 46, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 0, "jev_correct": 65, "local_accuracy": 0.0, "jev_accuracy": 0.65, "difference_pp": -65.0, "paired_cluster_bootstrap95_pp": [ -75.0, -56.0 ], "local_only_correct": 0, "jev_only_correct": 65, "both_correct": 0, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 39, "jev_correct": 39, "local_accuracy": 0.39, "jev_accuracy": 0.39, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -11.0, 12.0 ], "local_only_correct": 16, "jev_only_correct": 16, "both_correct": 23, "both_wrong": 45, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 5, "jev_correct": 9, "local_accuracy": 0.05, "jev_accuracy": 0.09, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -11.0, 3.0 ], "local_only_correct": 4, "jev_only_correct": 8, "both_correct": 1, "both_wrong": 87, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 24, "jev_correct": 59, "local_accuracy": 0.24, "jev_accuracy": 0.59, "difference_pp": -35.0, "paired_cluster_bootstrap95_pp": [ -45.0, -26.0 ], "local_only_correct": 0, "jev_only_correct": 35, "both_correct": 24, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 95, "jev_correct": 91, "local_accuracy": 0.95, "jev_accuracy": 0.91, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -3.0, 11.0 ], "local_only_correct": 8, "jev_only_correct": 4, "both_correct": 87, "both_wrong": 1, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 0, "jev_correct": 88, "local_accuracy": 0.0, "jev_accuracy": 0.88, "difference_pp": -88.0, "paired_cluster_bootstrap95_pp": [ -94.0, -81.0 ], "local_only_correct": 0, "jev_only_correct": 88, "both_correct": 0, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 88, "jev_correct": 48, "local_accuracy": 0.88, "jev_accuracy": 0.48, "difference_pp": 40.0, "paired_cluster_bootstrap95_pp": [ 29.0, 51.0 ], "local_only_correct": 44, "jev_only_correct": 4, "both_correct": 44, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 1256, "jev_correct": 2550, "local_accuracy": 0.4260515603799186, "jev_accuracy": 0.8649932157394844, "difference_pp": -43.89416553595658, "paired_cluster_bootstrap95_pp": [ -45.827679782903665, -42.06241519674356 ], "local_only_correct": 45, "jev_only_correct": 1339, "both_correct": 1211, "both_wrong": 353, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 1174, "jev_correct": 2412, "local_accuracy": 0.39823609226594303, "jev_accuracy": 0.8181818181818182, "difference_pp": -41.99457259158751, "paired_cluster_bootstrap95_pp": [ -44.02985074626866, -39.959294436906376 ], "local_only_correct": 118, "jev_only_correct": 1356, "both_correct": 1056, "both_wrong": 418, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 541, "jev_correct": 843, "local_accuracy": 0.46121057118499575, "jev_accuracy": 0.7186700767263428, "difference_pp": -25.7459505541347, "paired_cluster_bootstrap95_pp": [ -30.337368008689612, -21.235390020179516 ], "local_only_correct": 149, "jev_only_correct": 451, "both_correct": 392, "both_wrong": 181, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "kev-08b": { "fresh/news": { "n": 200, "local_correct": 177, "jev_correct": 178, "local_accuracy": 0.885, "jev_accuracy": 0.89, "difference_pp": -0.5, "paired_cluster_bootstrap95_pp": [ -5.0, 4.5 ], "local_only_correct": 12, "jev_only_correct": 13, "both_correct": 165, "both_wrong": 10, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 107, "jev_correct": 120, "local_accuracy": 0.535, "jev_accuracy": 0.6, "difference_pp": -6.5, "paired_cluster_bootstrap95_pp": [ -13.0, 0.012499999999931788 ], "local_only_correct": 17, "jev_only_correct": 30, "both_correct": 90, "both_wrong": 63, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 139, "jev_correct": 196, "local_accuracy": 0.695, "jev_accuracy": 0.98, "difference_pp": -28.499999999999996, "paired_cluster_bootstrap95_pp": [ -35.0, -22.5 ], "local_only_correct": 0, "jev_only_correct": 57, "both_correct": 139, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 132, "jev_correct": 179, "local_accuracy": 0.66, "jev_accuracy": 0.895, "difference_pp": -23.5, "paired_cluster_bootstrap95_pp": [ -30.0, -17.0 ], "local_only_correct": 4, "jev_only_correct": 51, "both_correct": 128, "both_wrong": 17, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 8, "jev_correct": 11, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9166666666666666, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 8, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 4, "jev_correct": 10, "local_accuracy": 0.3333333333333333, "jev_accuracy": 0.8333333333333334, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -83.33333333333333, -8.333333333333334 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 3, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 11, "jev_correct": 12, "local_accuracy": 0.9166666666666666, "jev_accuracy": 1.0, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 11, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 11, "jev_correct": 12, "local_accuracy": 0.9166666666666666, "jev_accuracy": 1.0, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 11, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 92, "jev_correct": 91, "local_accuracy": 0.92, "jev_accuracy": 0.91, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -6.0, 8.0 ], "local_only_correct": 7, "jev_only_correct": 6, "both_correct": 85, "both_wrong": 2, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 45, "jev_correct": 48, "local_accuracy": 0.45, "jev_accuracy": 0.48, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -14.0, 8.0 ], "local_only_correct": 13, "jev_only_correct": 16, "both_correct": 32, "both_wrong": 39, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 87, "jev_correct": 89, "local_accuracy": 0.87, "jev_accuracy": 0.89, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -8.0, 4.0 ], "local_only_correct": 4, "jev_only_correct": 6, "both_correct": 83, "both_wrong": 7, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 41, "jev_correct": 81, "local_accuracy": 0.36936936936936937, "jev_accuracy": 0.7297297297297297, "difference_pp": -36.03603603603604, "paired_cluster_bootstrap95_pp": [ -46.846846846846844, -25.225225225225227 ], "local_only_correct": 5, "jev_only_correct": 45, "both_correct": 36, "both_wrong": 25, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 58, "jev_correct": 70, "local_accuracy": 0.8055555555555556, "jev_accuracy": 0.9722222222222222, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, -5.555555555555555 ], "local_only_correct": 2, "jev_only_correct": 14, "both_correct": 56, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7368421052631579, "jev_accuracy": 0.9473684210526315, "difference_pp": -21.052631578947366, "paired_cluster_bootstrap95_pp": [ -42.10526315789474, -5.2631578947368425 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.8125, "jev_accuracy": 1.0, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 16, "jev_correct": 20, "local_accuracy": 0.7272727272727273, "jev_accuracy": 0.9090909090909091, "difference_pp": -18.181818181818183, "paired_cluster_bootstrap95_pp": [ -36.36363636363637, -4.545454545454546 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 16, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.8235294117647058, "jev_accuracy": 0.9411764705882353, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 11.764705882352942 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -60.0, -13.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.75, "jev_accuracy": 0.9375, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.65624999999999, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 9, "jev_correct": 18, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -72.22222222222223, -27.77777777777778 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 9, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 15, "jev_correct": 21, "local_accuracy": 0.6818181818181818, "jev_accuracy": 0.9545454545454546, "difference_pp": -27.27272727272727, "paired_cluster_bootstrap95_pp": [ -45.56818181818181, -9.090909090909092 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 15, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9444444444444444, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5, "jev_accuracy": 0.9375, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7777777777777778, "jev_accuracy": 0.9444444444444444, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -38.888888888888886, 5.555555555555555 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7, "jev_accuracy": 0.9, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, -5.0 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 5, "jev_correct": 18, "local_accuracy": 0.23809523809523808, "jev_accuracy": 0.8571428571428571, "difference_pp": -61.904761904761905, "paired_cluster_bootstrap95_pp": [ -85.71428571428571, -33.333333333333336 ], "local_only_correct": 1, "jev_only_correct": 14, "both_correct": 4, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 10, "jev_correct": 14, "local_accuracy": 0.7142857142857143, "jev_accuracy": 1.0, "difference_pp": -28.57142857142857, "paired_cluster_bootstrap95_pp": [ -50.0, -7.142857142857143 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 10, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.75, "jev_accuracy": 0.875, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -37.5, 12.5 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 8, "jev_correct": 14, "local_accuracy": 0.5, "jev_accuracy": 0.875, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -18.75 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 8, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.5625, "jev_accuracy": 1.0, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.8125, "jev_accuracy": 1.0, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -43.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7777777777777778, "jev_accuracy": 1.0, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 7, "jev_correct": 17, "local_accuracy": 0.4117647058823529, "jev_accuracy": 1.0, "difference_pp": -58.82352941176471, "paired_cluster_bootstrap95_pp": [ -82.3529411764706, -35.294117647058826 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 7, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.7222222222222222, "jev_accuracy": 1.0, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9444444444444444, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4666666666666667, "jev_accuracy": 1.0, "difference_pp": -53.333333333333336, "paired_cluster_bootstrap95_pp": [ -80.0, -26.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.5789473684210527, "jev_accuracy": 0.8947368421052632, "difference_pp": -31.57894736842105, "paired_cluster_bootstrap95_pp": [ -57.89473684210526, -5.2631578947368425 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 8, "jev_correct": 19, "local_accuracy": 0.4, "jev_accuracy": 0.95, "difference_pp": -55.00000000000001, "paired_cluster_bootstrap95_pp": [ -75.0, -35.0 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 8, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 7, "jev_correct": 16, "local_accuracy": 0.35, "jev_accuracy": 0.8, "difference_pp": -45.0, "paired_cluster_bootstrap95_pp": [ -70.0, -20.0 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 6, "both_wrong": 3, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 12, "jev_correct": 20, "local_accuracy": 0.5454545454545454, "jev_accuracy": 0.9090909090909091, "difference_pp": -36.36363636363637, "paired_cluster_bootstrap95_pp": [ -63.63636363636363, -9.090909090909092 ], "local_only_correct": 2, "jev_only_correct": 10, "both_correct": 10, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 517, "jev_correct": 629, "local_accuracy": 0.5744444444444444, "jev_accuracy": 0.6988888888888889, "difference_pp": -12.444444444444445, "paired_cluster_bootstrap95_pp": [ -21.024617670599127, -4.351590856436791 ], "local_only_correct": 108, "jev_only_correct": 220, "both_correct": 409, "both_wrong": 163, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 442, "jev_correct": 465, "local_accuracy": 0.7700348432055749, "jev_accuracy": 0.8101045296167247, "difference_pp": -4.006968641114982, "paired_cluster_bootstrap95_pp": [ -7.839721254355401, -0.34843205574912894 ], "local_only_correct": 46, "jev_only_correct": 69, "both_correct": 396, "both_wrong": 63, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 378, "jev_correct": 490, "local_accuracy": 0.54, "jev_accuracy": 0.7, "difference_pp": -16.0, "paired_cluster_bootstrap95_pp": [ -20.917655691810587, -11.527244292677327 ], "local_only_correct": 86, "jev_only_correct": 198, "both_correct": 292, "both_wrong": 124, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 19, "jev_correct": 97, "local_accuracy": 0.0475, "jev_accuracy": 0.2425, "difference_pp": -19.5, "paired_cluster_bootstrap95_pp": [ -24.25, -14.993750000000034 ], "local_only_correct": 15, "jev_only_correct": 93, "both_correct": 4, "both_wrong": 288, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 23, "jev_correct": 28, "local_accuracy": 0.8214285714285714, "jev_accuracy": 1.0, "difference_pp": -17.857142857142858, "paired_cluster_bootstrap95_pp": [ -32.142857142857146, -3.5714285714285716 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 23, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 33, "jev_correct": 40, "local_accuracy": 0.825, "jev_accuracy": 1.0, "difference_pp": -17.5, "paired_cluster_bootstrap95_pp": [ -30.0, -7.5 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 33, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 33, "jev_correct": 40, "local_accuracy": 0.825, "jev_accuracy": 1.0, "difference_pp": -17.5, "paired_cluster_bootstrap95_pp": [ -30.0, -7.5 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 33, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 28, "jev_correct": 39, "local_accuracy": 0.7, "jev_accuracy": 0.975, "difference_pp": -27.500000000000004, "paired_cluster_bootstrap95_pp": [ -42.5, -15.0 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 28, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 29, "jev_correct": 39, "local_accuracy": 0.725, "jev_accuracy": 0.975, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -40.0, -10.0 ], "local_only_correct": 1, "jev_only_correct": 11, "both_correct": 28, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 220, "jev_correct": 237, "local_accuracy": 0.55, "jev_accuracy": 0.5925, "difference_pp": -4.25, "paired_cluster_bootstrap95_pp": [ -8.5, 0.25 ], "local_only_correct": 30, "jev_only_correct": 47, "both_correct": 190, "both_wrong": 133, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 243, "jev_correct": 277, "local_accuracy": 0.81, "jev_accuracy": 0.9233333333333333, "difference_pp": -11.333333333333332, "paired_cluster_bootstrap95_pp": [ -15.333333333333334, -7.658333333333379 ], "local_only_correct": 4, "jev_only_correct": 38, "both_correct": 239, "both_wrong": 19, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 356, "jev_correct": 377, "local_accuracy": 0.89, "jev_accuracy": 0.9425, "difference_pp": -5.25, "paired_cluster_bootstrap95_pp": [ -8.5, -2.243750000000034 ], "local_only_correct": 10, "jev_only_correct": 31, "both_correct": 346, "both_wrong": 13, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 358, "jev_correct": 346, "local_accuracy": 0.895, "jev_accuracy": 0.865, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -0.25, 6.25 ], "local_only_correct": 28, "jev_only_correct": 16, "both_correct": 330, "both_wrong": 26, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 265, "jev_correct": 360, "local_accuracy": 0.6625, "jev_accuracy": 0.9, "difference_pp": -23.75, "paired_cluster_bootstrap95_pp": [ -28.75, -19.0 ], "local_only_correct": 14, "jev_only_correct": 109, "both_correct": 251, "both_wrong": 26, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 156, "jev_correct": 225, "local_accuracy": 0.52, "jev_accuracy": 0.75, "difference_pp": -23.0, "paired_cluster_bootstrap95_pp": [ -28.666666666666668, -17.333333333333332 ], "local_only_correct": 10, "jev_only_correct": 79, "both_correct": 146, "both_wrong": 65, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 277, "jev_correct": 393, "local_accuracy": 0.6925, "jev_accuracy": 0.9825, "difference_pp": -28.999999999999996, "paired_cluster_bootstrap95_pp": [ -33.5, -24.75 ], "local_only_correct": 0, "jev_only_correct": 116, "both_correct": 277, "both_wrong": 7, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 281, "jev_correct": 265, "local_accuracy": 0.7025, "jev_accuracy": 0.6625, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -0.75, 8.5 ], "local_only_correct": 51, "jev_only_correct": 35, "both_correct": 230, "both_wrong": 84, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 67, "jev_correct": 75, "local_accuracy": 0.67, "jev_accuracy": 0.75, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -21.0, 5.0 ], "local_only_correct": 18, "jev_only_correct": 26, "both_correct": 49, "both_wrong": 7, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 45, "jev_correct": 56, "local_accuracy": 0.45, "jev_accuracy": 0.56, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -20.0, -2.0 ], "local_only_correct": 7, "jev_only_correct": 18, "both_correct": 38, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 47, "jev_correct": 49, "local_accuracy": 0.47, "jev_accuracy": 0.49, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -13.0, 9.0 ], "local_only_correct": 13, "jev_only_correct": 15, "both_correct": 34, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 37, "jev_correct": 48, "local_accuracy": 0.37, "jev_accuracy": 0.48, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -20.0, -2.0 ], "local_only_correct": 5, "jev_only_correct": 16, "both_correct": 32, "both_wrong": 47, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 74, "jev_correct": 79, "local_accuracy": 0.74, "jev_accuracy": 0.79, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -11.0, 1.0 ], "local_only_correct": 3, "jev_only_correct": 8, "both_correct": 71, "both_wrong": 18, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 7, "jev_correct": 6, "local_accuracy": 0.07, "jev_accuracy": 0.06, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -3.0, 5.0 ], "local_only_correct": 3, "jev_only_correct": 2, "both_correct": 4, "both_wrong": 91, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 53, "jev_correct": 64, "local_accuracy": 0.53, "jev_accuracy": 0.64, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -20.0, -3.0 ], "local_only_correct": 5, "jev_only_correct": 16, "both_correct": 48, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 32, "jev_correct": 37, "local_accuracy": 0.32, "jev_accuracy": 0.37, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -14.0, 4.0 ], "local_only_correct": 7, "jev_only_correct": 12, "both_correct": 25, "both_wrong": 56, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 7, "jev_correct": 16, "local_accuracy": 0.07, "jev_accuracy": 0.16, "difference_pp": -9.0, "paired_cluster_bootstrap95_pp": [ -15.0, -3.0 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 6, "both_wrong": 83, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 12, "jev_correct": 34, "local_accuracy": 0.12, "jev_accuracy": 0.34, "difference_pp": -22.0, "paired_cluster_bootstrap95_pp": [ -33.0, -12.0 ], "local_only_correct": 6, "jev_only_correct": 28, "both_correct": 6, "both_wrong": 60, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 86, "jev_correct": 83, "local_accuracy": 0.86, "jev_accuracy": 0.83, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -3.0, 9.0 ], "local_only_correct": 7, "jev_only_correct": 4, "both_correct": 79, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 15, "jev_correct": 21, "local_accuracy": 0.15, "jev_accuracy": 0.21, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -16.0, 4.0 ], "local_only_correct": 11, "jev_only_correct": 17, "both_correct": 4, "both_wrong": 68, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 39, "jev_correct": 51, "local_accuracy": 0.39, "jev_accuracy": 0.51, "difference_pp": -12.0, "paired_cluster_bootstrap95_pp": [ -22.0, -2.0 ], "local_only_correct": 6, "jev_only_correct": 18, "both_correct": 33, "both_wrong": 43, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 52, "jev_correct": 65, "local_accuracy": 0.52, "jev_accuracy": 0.65, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -22.0, -5.0 ], "local_only_correct": 4, "jev_only_correct": 17, "both_correct": 48, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 37, "jev_correct": 39, "local_accuracy": 0.37, "jev_accuracy": 0.39, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -10.0, 6.0 ], "local_only_correct": 8, "jev_only_correct": 10, "both_correct": 29, "both_wrong": 53, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 7, "jev_correct": 9, "local_accuracy": 0.07, "jev_accuracy": 0.09, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -9.0, 5.0 ], "local_only_correct": 6, "jev_only_correct": 8, "both_correct": 1, "both_wrong": 85, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 53, "jev_correct": 59, "local_accuracy": 0.53, "jev_accuracy": 0.59, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -16.0, 4.0 ], "local_only_correct": 10, "jev_only_correct": 16, "both_correct": 43, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 95, "jev_correct": 91, "local_accuracy": 0.95, "jev_accuracy": 0.91, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -3.0, 11.0 ], "local_only_correct": 8, "jev_only_correct": 4, "both_correct": 87, "both_wrong": 1, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 81, "jev_correct": 88, "local_accuracy": 0.81, "jev_accuracy": 0.88, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 4, "jev_only_correct": 11, "both_correct": 77, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 43, "jev_correct": 48, "local_accuracy": 0.43, "jev_accuracy": 0.48, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -17.0, 7.0 ], "local_only_correct": 15, "jev_only_correct": 20, "both_correct": 28, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2114, "jev_correct": 2550, "local_accuracy": 0.7170963364993216, "jev_accuracy": 0.8649932157394844, "difference_pp": -14.789687924016281, "paired_cluster_bootstrap95_pp": [ -16.21438263229308, -13.22930800542741 ], "local_only_correct": 77, "jev_only_correct": 513, "both_correct": 2037, "both_wrong": 321, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 1719, "jev_correct": 2412, "local_accuracy": 0.5831071913161465, "jev_accuracy": 0.8181818181818182, "difference_pp": -23.507462686567166, "paired_cluster_bootstrap95_pp": [ -25.27137042062415, -21.743554952510177 ], "local_only_correct": 77, "jev_only_correct": 770, "both_correct": 1642, "both_wrong": 459, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 757, "jev_correct": 843, "local_accuracy": 0.6453537936913896, "jev_accuracy": 0.7186700767263428, "difference_pp": -7.331628303495311, "paired_cluster_bootstrap95_pp": [ -11.630756762799017, -3.407247084337113 ], "local_only_correct": 164, "jev_only_correct": 250, "both_correct": 593, "both_wrong": 166, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "kev-4b": { "fresh/news": { "n": 200, "local_correct": 179, "jev_correct": 178, "local_accuracy": 0.895, "jev_accuracy": 0.89, "difference_pp": 0.5, "paired_cluster_bootstrap95_pp": [ -3.5, 4.5 ], "local_only_correct": 9, "jev_only_correct": 8, "both_correct": 170, "both_wrong": 13, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 114, "jev_correct": 120, "local_accuracy": 0.57, "jev_accuracy": 0.6, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -8.5, 2.5 ], "local_only_correct": 13, "jev_only_correct": 19, "both_correct": 101, "both_wrong": 67, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 186, "jev_correct": 196, "local_accuracy": 0.93, "jev_accuracy": 0.98, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -8.5, -2.0 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 186, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 182, "jev_correct": 179, "local_accuracy": 0.91, "jev_accuracy": 0.895, "difference_pp": 1.5, "paired_cluster_bootstrap95_pp": [ -1.5, 4.5 ], "local_only_correct": 6, "jev_only_correct": 3, "both_correct": 176, "both_wrong": 15, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 9, "jev_correct": 11, "local_accuracy": 0.75, "jev_accuracy": 0.9166666666666666, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 9, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 8, "jev_correct": 10, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.8333333333333334, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 8, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 95, "jev_correct": 91, "local_accuracy": 0.95, "jev_accuracy": 0.91, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -3.0, 11.0 ], "local_only_correct": 8, "jev_only_correct": 4, "both_correct": 87, "both_wrong": 1, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 45, "jev_correct": 48, "local_accuracy": 0.45, "jev_accuracy": 0.48, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -11.0, 6.0 ], "local_only_correct": 8, "jev_only_correct": 11, "both_correct": 37, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 88, "jev_correct": 89, "local_accuracy": 0.88, "jev_accuracy": 0.89, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 5.0 ], "local_only_correct": 4, "jev_only_correct": 5, "both_correct": 84, "both_wrong": 7, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 60, "jev_correct": 81, "local_accuracy": 0.5405405405405406, "jev_accuracy": 0.7297297297297297, "difference_pp": -18.91891891891892, "paired_cluster_bootstrap95_pp": [ -27.92792792792793, -10.81081081081081 ], "local_only_correct": 3, "jev_only_correct": 24, "both_correct": 57, "both_wrong": 27, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 67, "jev_correct": 70, "local_accuracy": 0.9305555555555556, "jev_accuracy": 0.9722222222222222, "difference_pp": -4.166666666666666, "paired_cluster_bootstrap95_pp": [ -13.88888888888889, 4.166666666666667 ], "local_only_correct": 2, "jev_only_correct": 5, "both_correct": 65, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.8947368421052632, "jev_accuracy": 0.9473684210526315, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 21, "jev_correct": 20, "local_accuracy": 0.9545454545454546, "jev_accuracy": 0.9090909090909091, "difference_pp": 4.545454545454546, "paired_cluster_bootstrap95_pp": [ -9.090909090909092, 22.727272727272727 ], "local_only_correct": 2, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.8235294117647058, "jev_accuracy": 0.9411764705882353, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 11.764705882352942 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 18, "jev_correct": 21, "local_accuracy": 0.8181818181818182, "jev_accuracy": 0.9545454545454546, "difference_pp": -13.636363636363635, "paired_cluster_bootstrap95_pp": [ -31.818181818181817, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 18, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7222222222222222, "jev_accuracy": 0.9444444444444444, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, 0.0 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 17, "jev_correct": 17, "local_accuracy": 0.9444444444444444, "jev_accuracy": 0.9444444444444444, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 16.666666666666668 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8, "jev_accuracy": 0.9, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 11, "jev_correct": 18, "local_accuracy": 0.5238095238095238, "jev_accuracy": 0.8571428571428571, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -57.142857142857146, -9.523809523809524 ], "local_only_correct": 1, "jev_only_correct": 8, "both_correct": 10, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.8571428571428571, "jev_accuracy": 1.0, "difference_pp": -14.285714285714285, "paired_cluster_bootstrap95_pp": [ -35.714285714285715, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 15, "jev_correct": 14, "local_accuracy": 0.9375, "jev_accuracy": 0.875, "difference_pp": 6.25, "paired_cluster_bootstrap95_pp": [ -12.5, 25.0 ], "local_only_correct": 2, "jev_only_correct": 1, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.6875, "jev_accuracy": 0.875, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7777777777777778, "jev_accuracy": 0.9444444444444444, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -38.888888888888886, 5.555555555555555 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8125, "jev_accuracy": 0.9375, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -37.5, 12.5 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.7333333333333333, "jev_accuracy": 1.0, "difference_pp": -26.666666666666668, "paired_cluster_bootstrap95_pp": [ -46.666666666666664, -6.666666666666667 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 17, "jev_correct": 17, "local_accuracy": 0.8947368421052632, "jev_accuracy": 0.8947368421052632, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 15.789473684210526 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9, "jev_accuracy": 0.95, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -25.0, 10.0 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 17, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.6, "jev_accuracy": 0.8, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, -5.0 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 4, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 18, "jev_correct": 20, "local_accuracy": 0.8181818181818182, "jev_accuracy": 0.9090909090909091, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -27.272727272727273, 9.090909090909092 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 17, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 640, "jev_correct": 629, "local_accuracy": 0.7111111111111111, "jev_accuracy": 0.6988888888888889, "difference_pp": 1.2222222222222223, "paired_cluster_bootstrap95_pp": [ -3.7713463843857493, 5.729198194108152 ], "local_only_correct": 110, "jev_only_correct": 99, "both_correct": 530, "both_wrong": 161, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 488, "jev_correct": 465, "local_accuracy": 0.8501742160278746, "jev_accuracy": 0.8101045296167247, "difference_pp": 4.006968641114982, "paired_cluster_bootstrap95_pp": [ 0.8710801393728222, 7.142857142857143 ], "local_only_correct": 54, "jev_only_correct": 31, "both_correct": 434, "both_wrong": 55, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 507, "jev_correct": 490, "local_accuracy": 0.7242857142857143, "jev_accuracy": 0.7, "difference_pp": 2.4285714285714284, "paired_cluster_bootstrap95_pp": [ -1.5761583115199458, 6.313400489017314 ], "local_only_correct": 107, "jev_only_correct": 90, "both_correct": 400, "both_wrong": 103, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 60, "jev_correct": 97, "local_accuracy": 0.15, "jev_accuracy": 0.2425, "difference_pp": -9.25, "paired_cluster_bootstrap95_pp": [ -14.25, -4.25 ], "local_only_correct": 38, "jev_only_correct": 75, "both_correct": 22, "both_wrong": 265, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 26, "jev_correct": 28, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -17.857142857142858, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 26, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 38, "jev_correct": 40, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 38, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 35, "jev_correct": 40, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -22.5, -2.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 35, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 37, "jev_correct": 39, "local_accuracy": 0.925, "jev_accuracy": 0.975, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 37, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 35, "jev_correct": 39, "local_accuracy": 0.875, "jev_accuracy": 0.975, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -20.0, -2.5 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 35, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 224, "jev_correct": 237, "local_accuracy": 0.56, "jev_accuracy": 0.5925, "difference_pp": -3.25, "paired_cluster_bootstrap95_pp": [ -6.75, 0.25 ], "local_only_correct": 19, "jev_only_correct": 32, "both_correct": 205, "both_wrong": 144, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 254, "jev_correct": 277, "local_accuracy": 0.8466666666666667, "jev_accuracy": 0.9233333333333333, "difference_pp": -7.666666666666666, "paired_cluster_bootstrap95_pp": [ -11.333333333333334, -4.333333333333333 ], "local_only_correct": 2, "jev_only_correct": 25, "both_correct": 252, "both_wrong": 21, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 294, "jev_correct": 377, "local_accuracy": 0.735, "jev_accuracy": 0.9425, "difference_pp": -20.75, "paired_cluster_bootstrap95_pp": [ -24.75, -16.75 ], "local_only_correct": 2, "jev_only_correct": 85, "both_correct": 292, "both_wrong": 21, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 370, "jev_correct": 346, "local_accuracy": 0.925, "jev_accuracy": 0.865, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ 3.25, 9.0 ], "local_only_correct": 29, "jev_only_correct": 5, "both_correct": 341, "both_wrong": 25, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 361, "jev_correct": 360, "local_accuracy": 0.9025, "jev_accuracy": 0.9, "difference_pp": 0.25, "paired_cluster_bootstrap95_pp": [ -2.5, 3.0 ], "local_only_correct": 17, "jev_only_correct": 16, "both_correct": 344, "both_wrong": 23, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 214, "jev_correct": 225, "local_accuracy": 0.7133333333333334, "jev_accuracy": 0.75, "difference_pp": -3.6666666666666665, "paired_cluster_bootstrap95_pp": [ -7.0, -0.3333333333333333 ], "local_only_correct": 9, "jev_only_correct": 20, "both_correct": 205, "both_wrong": 66, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 365, "jev_correct": 393, "local_accuracy": 0.9125, "jev_accuracy": 0.9825, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -9.5, -4.5 ], "local_only_correct": 1, "jev_only_correct": 29, "both_correct": 364, "both_wrong": 6, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 299, "jev_correct": 265, "local_accuracy": 0.7475, "jev_accuracy": 0.6625, "difference_pp": 8.5, "paired_cluster_bootstrap95_pp": [ 4.25, 12.75 ], "local_only_correct": 54, "jev_only_correct": 20, "both_correct": 245, "both_wrong": 81, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 74, "jev_correct": 75, "local_accuracy": 0.74, "jev_accuracy": 0.75, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -8.0, 6.0 ], "local_only_correct": 6, "jev_only_correct": 7, "both_correct": 68, "both_wrong": 19, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 43, "jev_correct": 56, "local_accuracy": 0.43, "jev_accuracy": 0.56, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -23.0, -3.0 ], "local_only_correct": 6, "jev_only_correct": 19, "both_correct": 37, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 46, "jev_correct": 49, "local_accuracy": 0.46, "jev_accuracy": 0.49, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -9.0, 3.0 ], "local_only_correct": 3, "jev_only_correct": 6, "both_correct": 43, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 38, "jev_correct": 48, "local_accuracy": 0.38, "jev_accuracy": 0.48, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -18.0, -2.0 ], "local_only_correct": 4, "jev_only_correct": 14, "both_correct": 34, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 74, "jev_correct": 79, "local_accuracy": 0.74, "jev_accuracy": 0.79, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -10.0, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 73, "both_wrong": 20, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 4, "jev_correct": 6, "local_accuracy": 0.04, "jev_accuracy": 0.06, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -6.0, 2.0 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 3, "both_wrong": 93, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 65, "jev_correct": 64, "local_accuracy": 0.65, "jev_accuracy": 0.64, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 9.0 ], "local_only_correct": 9, "jev_only_correct": 8, "both_correct": 56, "both_wrong": 27, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 37, "jev_correct": 37, "local_accuracy": 0.37, "jev_accuracy": 0.37, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -7.0, 7.0 ], "local_only_correct": 6, "jev_only_correct": 6, "both_correct": 31, "both_wrong": 57, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 8, "jev_correct": 16, "local_accuracy": 0.08, "jev_accuracy": 0.16, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -14.0, -3.0 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 7, "both_wrong": 83, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 30, "jev_correct": 34, "local_accuracy": 0.3, "jev_accuracy": 0.34, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -12.0, 4.0 ], "local_only_correct": 6, "jev_only_correct": 10, "both_correct": 24, "both_wrong": 60, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 85, "jev_correct": 83, "local_accuracy": 0.85, "jev_accuracy": 0.83, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -4.0, 8.0 ], "local_only_correct": 6, "jev_only_correct": 4, "both_correct": 79, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 15, "jev_correct": 21, "local_accuracy": 0.15, "jev_accuracy": 0.21, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -15.0, 2.0 ], "local_only_correct": 6, "jev_only_correct": 12, "both_correct": 9, "both_wrong": 73, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 51, "jev_correct": 51, "local_accuracy": 0.51, "jev_accuracy": 0.51, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -6.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 4, "both_correct": 47, "both_wrong": 45, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 53, "jev_correct": 65, "local_accuracy": 0.53, "jev_accuracy": 0.65, "difference_pp": -12.0, "paired_cluster_bootstrap95_pp": [ -22.0, -3.0 ], "local_only_correct": 6, "jev_only_correct": 18, "both_correct": 47, "both_wrong": 29, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 36, "jev_correct": 39, "local_accuracy": 0.36, "jev_accuracy": 0.39, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -11.0, 5.0 ], "local_only_correct": 7, "jev_only_correct": 10, "both_correct": 29, "both_wrong": 54, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 12, "jev_correct": 9, "local_accuracy": 0.12, "jev_accuracy": 0.09, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -3.0, 9.0 ], "local_only_correct": 6, "jev_only_correct": 3, "both_correct": 6, "both_wrong": 85, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 59, "jev_correct": 59, "local_accuracy": 0.59, "jev_accuracy": 0.59, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -8.0, 7.0 ], "local_only_correct": 8, "jev_only_correct": 8, "both_correct": 51, "both_wrong": 33, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 95, "jev_correct": 91, "local_accuracy": 0.95, "jev_accuracy": 0.91, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -3.0, 11.0 ], "local_only_correct": 8, "jev_only_correct": 4, "both_correct": 87, "both_wrong": 1, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 88, "jev_correct": 88, "local_accuracy": 0.88, "jev_accuracy": 0.88, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -5.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 4, "both_correct": 84, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 48, "jev_correct": 48, "local_accuracy": 0.48, "jev_accuracy": 0.48, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -8.0, 8.0 ], "local_only_correct": 9, "jev_only_correct": 9, "both_correct": 39, "both_wrong": 43, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2327, "jev_correct": 2550, "local_accuracy": 0.7893487109905021, "jev_accuracy": 0.8649932157394844, "difference_pp": -7.564450474898236, "paired_cluster_bootstrap95_pp": [ -8.819538670284938, -6.343283582089552 ], "local_only_correct": 69, "jev_only_correct": 292, "both_correct": 2258, "both_wrong": 329, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2123, "jev_correct": 2412, "local_accuracy": 0.7201492537313433, "jev_accuracy": 0.8181818181818182, "difference_pp": -9.80325644504749, "paired_cluster_bootstrap95_pp": [ -11.12618724559023, -8.44640434192673 ], "local_only_correct": 73, "jev_only_correct": 362, "both_correct": 2050, "both_wrong": 463, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 867, "jev_correct": 843, "local_accuracy": 0.7391304347826086, "jev_accuracy": 0.7186700767263428, "difference_pp": 2.0460358056265986, "paired_cluster_bootstrap95_pp": [ -1.2394323555385258, 5.1762609649122755 ], "local_only_correct": 163, "jev_only_correct": 139, "both_correct": 704, "both_wrong": 167, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "laya-multilingual": { "fresh/news": { "n": 200, "local_correct": 183, "jev_correct": 178, "local_accuracy": 0.915, "jev_accuracy": 0.89, "difference_pp": 2.5, "paired_cluster_bootstrap95_pp": [ -1.5, 7.0 ], "local_only_correct": 12, "jev_only_correct": 7, "both_correct": 171, "both_wrong": 10, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 103, "jev_correct": 120, "local_accuracy": 0.515, "jev_accuracy": 0.6, "difference_pp": -8.5, "paired_cluster_bootstrap95_pp": [ -15.5, -1.5 ], "local_only_correct": 18, "jev_only_correct": 35, "both_correct": 85, "both_wrong": 62, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 200, "jev_correct": 196, "local_accuracy": 1.0, "jev_accuracy": 0.98, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ 0.5, 4.0 ], "local_only_correct": 4, "jev_only_correct": 0, "both_correct": 196, "both_wrong": 0, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 200, "jev_correct": 179, "local_accuracy": 1.0, "jev_accuracy": 0.895, "difference_pp": 10.5, "paired_cluster_bootstrap95_pp": [ 6.5, 15.0 ], "local_only_correct": 21, "jev_only_correct": 0, "both_correct": 179, "both_wrong": 0, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 6, "jev_correct": 11, "local_accuracy": 0.5, "jev_accuracy": 0.9166666666666666, "difference_pp": -41.66666666666667, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 6, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 6, "jev_correct": 10, "local_accuracy": 0.5, "jev_accuracy": 0.8333333333333334, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -75.0, 8.333333333333334 ], "local_only_correct": 2, "jev_only_correct": 6, "both_correct": 4, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 8, "jev_correct": 12, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -58.333333333333336, -8.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 8, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 10, "jev_correct": 12, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 10, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 91, "jev_correct": 91, "local_accuracy": 0.91, "jev_accuracy": 0.91, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -8.0, 8.0 ], "local_only_correct": 8, "jev_only_correct": 8, "both_correct": 83, "both_wrong": 1, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 22, "jev_correct": 48, "local_accuracy": 0.22, "jev_accuracy": 0.48, "difference_pp": -26.0, "paired_cluster_bootstrap95_pp": [ -37.0, -15.0 ], "local_only_correct": 6, "jev_only_correct": 32, "both_correct": 16, "both_wrong": 46, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 2, "jev_correct": 89, "local_accuracy": 0.02, "jev_accuracy": 0.89, "difference_pp": -87.0, "paired_cluster_bootstrap95_pp": [ -93.0, -80.0 ], "local_only_correct": 0, "jev_only_correct": 87, "both_correct": 2, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 43, "jev_correct": 48, "local_accuracy": 0.8958333333333334, "jev_accuracy": 1.0, "difference_pp": -10.416666666666668, "paired_cluster_bootstrap95_pp": [ -18.75, -2.0833333333333335 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 43, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/hard": { "n": 111, "local_correct": 37, "jev_correct": 81, "local_accuracy": 0.3333333333333333, "jev_accuracy": 0.7297297297297297, "difference_pp": -39.63963963963964, "paired_cluster_bootstrap95_pp": [ -51.351351351351354, -27.92792792792793 ], "local_only_correct": 10, "jev_only_correct": 54, "both_correct": 27, "both_wrong": 20, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 29, "jev_correct": 70, "local_accuracy": 0.4027777777777778, "jev_accuracy": 0.9722222222222222, "difference_pp": -56.94444444444444, "paired_cluster_bootstrap95_pp": [ -69.44444444444444, -43.05555555555556 ], "local_only_correct": 1, "jev_only_correct": 42, "both_correct": 28, "both_wrong": 1, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 11, "jev_correct": 18, "local_accuracy": 0.5789473684210527, "jev_accuracy": 0.9473684210526315, "difference_pp": -36.84210526315789, "paired_cluster_bootstrap95_pp": [ -63.1578947368421, -10.526315789473685 ], "local_only_correct": 1, "jev_only_correct": 8, "both_correct": 10, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 10, "jev_correct": 16, "local_accuracy": 0.625, "jev_accuracy": 1.0, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 10, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 12, "jev_correct": 20, "local_accuracy": 0.5454545454545454, "jev_accuracy": 0.9090909090909091, "difference_pp": -36.36363636363637, "paired_cluster_bootstrap95_pp": [ -63.63636363636363, -9.090909090909092 ], "local_only_correct": 2, "jev_only_correct": 10, "both_correct": 10, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 7, "jev_correct": 18, "local_accuracy": 0.3888888888888889, "jev_accuracy": 1.0, "difference_pp": -61.111111111111114, "paired_cluster_bootstrap95_pp": [ -83.33333333333333, -38.888888888888886 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 7, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 17, "jev_correct": 19, "local_accuracy": 0.8947368421052632, "jev_accuracy": 1.0, "difference_pp": -10.526315789473683, "paired_cluster_bootstrap95_pp": [ -26.31578947368421, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 17, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 4, "jev_correct": 16, "local_accuracy": 0.23529411764705882, "jev_accuracy": 0.9411764705882353, "difference_pp": -70.58823529411765, "paired_cluster_bootstrap95_pp": [ -88.38235294117646, -47.05882352941177 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 4, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4666666666666667, "jev_accuracy": 1.0, "difference_pp": -53.333333333333336, "paired_cluster_bootstrap95_pp": [ -80.0, -26.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 6, "jev_correct": 15, "local_accuracy": 0.375, "jev_accuracy": 0.9375, "difference_pp": -56.25, "paired_cluster_bootstrap95_pp": [ -87.5, -25.0 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 5, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 10, "jev_correct": 18, "local_accuracy": 0.5555555555555556, "jev_accuracy": 1.0, "difference_pp": -44.44444444444444, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -22.22222222222222 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 10, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 12, "jev_correct": 21, "local_accuracy": 0.5454545454545454, "jev_accuracy": 0.9545454545454546, "difference_pp": -40.909090909090914, "paired_cluster_bootstrap95_pp": [ -59.09090909090909, -22.727272727272727 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 12, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 15, "jev_correct": 19, "local_accuracy": 0.7894736842105263, "jev_accuracy": 1.0, "difference_pp": -21.052631578947366, "paired_cluster_bootstrap95_pp": [ -42.10526315789474, -5.2631578947368425 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 15, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 9, "jev_correct": 15, "local_accuracy": 0.6, "jev_accuracy": 1.0, "difference_pp": -40.0, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -20.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 9, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.7222222222222222, "jev_accuracy": 1.0, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5555555555555556, "jev_accuracy": 0.9444444444444444, "difference_pp": -38.88888888888889, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 4, "jev_correct": 15, "local_accuracy": 0.25, "jev_accuracy": 0.9375, "difference_pp": -68.75, "paired_cluster_bootstrap95_pp": [ -87.5, -43.75 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 4, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.4444444444444444, "jev_accuracy": 0.9444444444444444, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -72.22222222222223, -27.77777777777778 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 9, "jev_correct": 18, "local_accuracy": 0.45, "jev_accuracy": 0.9, "difference_pp": -45.0, "paired_cluster_bootstrap95_pp": [ -70.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 9, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 5, "jev_correct": 18, "local_accuracy": 0.23809523809523808, "jev_accuracy": 0.8571428571428571, "difference_pp": -61.904761904761905, "paired_cluster_bootstrap95_pp": [ -85.71428571428571, -33.333333333333336 ], "local_only_correct": 1, "jev_only_correct": 14, "both_correct": 4, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 7, "jev_correct": 16, "local_accuracy": 0.4375, "jev_accuracy": 1.0, "difference_pp": -56.25, "paired_cluster_bootstrap95_pp": [ -81.25, -31.25 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 7, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 18, "jev_correct": 23, "local_accuracy": 0.782608695652174, "jev_accuracy": 1.0, "difference_pp": -21.73913043478261, "paired_cluster_bootstrap95_pp": [ -39.130434782608695, -4.3478260869565215 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 18, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 7, "jev_correct": 14, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -78.57142857142857, -21.428571428571427 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 7, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 5, "jev_correct": 14, "local_accuracy": 0.3125, "jev_accuracy": 0.875, "difference_pp": -56.25, "paired_cluster_bootstrap95_pp": [ -87.5, -18.75 ], "local_only_correct": 2, "jev_only_correct": 11, "both_correct": 3, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 8, "jev_correct": 14, "local_accuracy": 0.5, "jev_accuracy": 0.875, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 8, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 11, "jev_correct": 16, "local_accuracy": 0.6875, "jev_accuracy": 1.0, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 11, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 8, "jev_correct": 16, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 8, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 5, "jev_correct": 18, "local_accuracy": 0.2777777777777778, "jev_accuracy": 1.0, "difference_pp": -72.22222222222221, "paired_cluster_bootstrap95_pp": [ -88.88888888888889, -50.0 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 5, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 4, "jev_correct": 17, "local_accuracy": 0.23529411764705882, "jev_accuracy": 1.0, "difference_pp": -76.47058823529412, "paired_cluster_bootstrap95_pp": [ -94.11764705882354, -52.94117647058823 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 4, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 5, "jev_correct": 18, "local_accuracy": 0.2777777777777778, "jev_accuracy": 1.0, "difference_pp": -72.22222222222221, "paired_cluster_bootstrap95_pp": [ -88.88888888888889, -50.0 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 5, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 4, "jev_correct": 17, "local_accuracy": 0.2222222222222222, "jev_accuracy": 0.9444444444444444, "difference_pp": -72.22222222222221, "paired_cluster_bootstrap95_pp": [ -94.44444444444444, -50.0 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 4, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5, "jev_accuracy": 0.9375, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 4, "jev_correct": 15, "local_accuracy": 0.26666666666666666, "jev_accuracy": 1.0, "difference_pp": -73.33333333333333, "paired_cluster_bootstrap95_pp": [ -93.33333333333333, -53.333333333333336 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 4, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.42105263157894735, "jev_accuracy": 0.8947368421052632, "difference_pp": -47.368421052631575, "paired_cluster_bootstrap95_pp": [ -73.6842105263158, -21.05263157894737 ], "local_only_correct": 1, "jev_only_correct": 10, "both_correct": 7, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 8, "jev_correct": 18, "local_accuracy": 0.4444444444444444, "jev_accuracy": 1.0, "difference_pp": -55.55555555555556, "paired_cluster_bootstrap95_pp": [ -77.77777777777777, -33.333333333333336 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 8, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 9, "jev_correct": 19, "local_accuracy": 0.45, "jev_accuracy": 0.95, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 1, "jev_only_correct": 11, "both_correct": 8, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 14, "jev_correct": 20, "local_accuracy": 0.7, "jev_accuracy": 1.0, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -50.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 14, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 7, "jev_correct": 16, "local_accuracy": 0.35, "jev_accuracy": 0.8, "difference_pp": -45.0, "paired_cluster_bootstrap95_pp": [ -65.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 7, "both_wrong": 4, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 7, "jev_correct": 20, "local_accuracy": 0.3181818181818182, "jev_accuracy": 0.9090909090909091, "difference_pp": -59.09090909090909, "paired_cluster_bootstrap95_pp": [ -77.27272727272727, -36.36363636363637 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 7, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 411, "jev_correct": 629, "local_accuracy": 0.45666666666666667, "jev_accuracy": 0.6988888888888889, "difference_pp": -24.22222222222222, "paired_cluster_bootstrap95_pp": [ -31.077875134968984, -18.155876517424673 ], "local_only_correct": 114, "jev_only_correct": 332, "both_correct": 297, "both_wrong": 157, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 175, "jev_correct": 465, "local_accuracy": 0.3048780487804878, "jev_accuracy": 0.8101045296167247, "difference_pp": -50.522648083623686, "paired_cluster_bootstrap95_pp": [ -55.22648083623693, -46.167247386759584 ], "local_only_correct": 11, "jev_only_correct": 301, "both_correct": 164, "both_wrong": 98, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 116, "jev_correct": 490, "local_accuracy": 0.1657142857142857, "jev_accuracy": 0.7, "difference_pp": -53.42857142857142, "paired_cluster_bootstrap95_pp": [ -57.71498862931569, -49.05286302970663 ], "local_only_correct": 35, "jev_only_correct": 409, "both_correct": 81, "both_wrong": 175, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 1, "jev_correct": 97, "local_accuracy": 0.0025, "jev_accuracy": 0.2425, "difference_pp": -24.0, "paired_cluster_bootstrap95_pp": [ -28.0, -19.75 ], "local_only_correct": 0, "jev_only_correct": 96, "both_correct": 1, "both_wrong": 303, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 23, "jev_correct": 28, "local_accuracy": 0.8214285714285714, "jev_accuracy": 1.0, "difference_pp": -17.857142857142858, "paired_cluster_bootstrap95_pp": [ -32.142857142857146, -3.5714285714285716 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 23, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 21, "jev_correct": 28, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, -10.714285714285714 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 21, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 22, "jev_correct": 28, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -35.714285714285715, -7.142857142857143 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 22, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 19, "jev_correct": 28, "local_accuracy": 0.6785714285714286, "jev_accuracy": 1.0, "difference_pp": -32.142857142857146, "paired_cluster_bootstrap95_pp": [ -50.0, -14.285714285714286 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 19, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 19, "jev_correct": 40, "local_accuracy": 0.475, "jev_accuracy": 1.0, "difference_pp": -52.5, "paired_cluster_bootstrap95_pp": [ -67.5, -37.5 ], "local_only_correct": 0, "jev_only_correct": 21, "both_correct": 19, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 20, "jev_correct": 40, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -65.0, -35.0 ], "local_only_correct": 0, "jev_only_correct": 20, "both_correct": 20, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 18, "jev_correct": 39, "local_accuracy": 0.45, "jev_accuracy": 0.975, "difference_pp": -52.5, "paired_cluster_bootstrap95_pp": [ -70.0, -37.5 ], "local_only_correct": 1, "jev_only_correct": 22, "both_correct": 17, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 18, "jev_correct": 39, "local_accuracy": 0.45, "jev_accuracy": 0.975, "difference_pp": -52.5, "paired_cluster_bootstrap95_pp": [ -67.5, -37.5 ], "local_only_correct": 0, "jev_only_correct": 21, "both_correct": 18, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 215, "jev_correct": 237, "local_accuracy": 0.5375, "jev_accuracy": 0.5925, "difference_pp": -5.5, "paired_cluster_bootstrap95_pp": [ -10.25, -0.75 ], "local_only_correct": 38, "jev_only_correct": 60, "both_correct": 177, "both_wrong": 125, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 196, "jev_correct": 277, "local_accuracy": 0.6533333333333333, "jev_accuracy": 0.9233333333333333, "difference_pp": -27.0, "paired_cluster_bootstrap95_pp": [ -32.333333333333336, -21.333333333333332 ], "local_only_correct": 6, "jev_only_correct": 87, "both_correct": 190, "both_wrong": 17, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 322, "jev_correct": 377, "local_accuracy": 0.805, "jev_accuracy": 0.9425, "difference_pp": -13.750000000000002, "paired_cluster_bootstrap95_pp": [ -17.75, -10.0 ], "local_only_correct": 7, "jev_only_correct": 62, "both_correct": 315, "both_wrong": 16, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 375, "jev_correct": 346, "local_accuracy": 0.9375, "jev_accuracy": 0.865, "difference_pp": 7.249999999999999, "paired_cluster_bootstrap95_pp": [ 3.75, 11.0 ], "local_only_correct": 43, "jev_only_correct": 14, "both_correct": 332, "both_wrong": 11, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 397, "jev_correct": 360, "local_accuracy": 0.9925, "jev_accuracy": 0.9, "difference_pp": 9.25, "paired_cluster_bootstrap95_pp": [ 6.0, 12.5 ], "local_only_correct": 40, "jev_only_correct": 3, "both_correct": 357, "both_wrong": 0, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 168, "jev_correct": 225, "local_accuracy": 0.56, "jev_accuracy": 0.75, "difference_pp": -19.0, "paired_cluster_bootstrap95_pp": [ -26.0, -12.0 ], "local_only_correct": 32, "jev_only_correct": 89, "both_correct": 136, "both_wrong": 43, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 399, "jev_correct": 393, "local_accuracy": 0.9975, "jev_accuracy": 0.9825, "difference_pp": 1.5, "paired_cluster_bootstrap95_pp": [ 0.25, 3.0 ], "local_only_correct": 7, "jev_only_correct": 1, "both_correct": 392, "both_wrong": 0, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 214, "jev_correct": 265, "local_accuracy": 0.535, "jev_accuracy": 0.6625, "difference_pp": -12.75, "paired_cluster_bootstrap95_pp": [ -16.75, -8.75 ], "local_only_correct": 13, "jev_only_correct": 64, "both_correct": 201, "both_wrong": 122, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 47, "jev_correct": 75, "local_accuracy": 0.47, "jev_accuracy": 0.75, "difference_pp": -28.000000000000004, "paired_cluster_bootstrap95_pp": [ -44.0, -11.0 ], "local_only_correct": 22, "jev_only_correct": 50, "both_correct": 25, "both_wrong": 3, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 38, "jev_correct": 56, "local_accuracy": 0.38, "jev_accuracy": 0.56, "difference_pp": -18.0, "paired_cluster_bootstrap95_pp": [ -29.0, -8.0 ], "local_only_correct": 6, "jev_only_correct": 24, "both_correct": 32, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 38, "jev_correct": 49, "local_accuracy": 0.38, "jev_accuracy": 0.49, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -24.0, 3.0 ], "local_only_correct": 19, "jev_only_correct": 30, "both_correct": 19, "both_wrong": 32, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 18, "jev_correct": 48, "local_accuracy": 0.18, "jev_accuracy": 0.48, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -41.0, -20.0 ], "local_only_correct": 4, "jev_only_correct": 34, "both_correct": 14, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 69, "jev_correct": 79, "local_accuracy": 0.69, "jev_accuracy": 0.79, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -17.0, -3.0 ], "local_only_correct": 2, "jev_only_correct": 12, "both_correct": 67, "both_wrong": 19, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 4, "jev_correct": 6, "local_accuracy": 0.04, "jev_accuracy": 0.06, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -8.0, 4.0 ], "local_only_correct": 4, "jev_only_correct": 6, "both_correct": 0, "both_wrong": 90, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 46, "jev_correct": 64, "local_accuracy": 0.46, "jev_accuracy": 0.64, "difference_pp": -18.0, "paired_cluster_bootstrap95_pp": [ -28.0, -8.0 ], "local_only_correct": 6, "jev_only_correct": 24, "both_correct": 40, "both_wrong": 30, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 31, "jev_correct": 37, "local_accuracy": 0.31, "jev_accuracy": 0.37, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -15.0, 4.0 ], "local_only_correct": 9, "jev_only_correct": 15, "both_correct": 22, "both_wrong": 54, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 1, "jev_correct": 16, "local_accuracy": 0.01, "jev_accuracy": 0.16, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -22.0, -9.0 ], "local_only_correct": 0, "jev_only_correct": 15, "both_correct": 1, "both_wrong": 84, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 1, "jev_correct": 34, "local_accuracy": 0.01, "jev_accuracy": 0.34, "difference_pp": -33.0, "paired_cluster_bootstrap95_pp": [ -43.0, -23.0 ], "local_only_correct": 1, "jev_only_correct": 34, "both_correct": 0, "both_wrong": 65, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 63, "jev_correct": 83, "local_accuracy": 0.63, "jev_accuracy": 0.83, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -28.0, -12.0 ], "local_only_correct": 1, "jev_only_correct": 21, "both_correct": 62, "both_wrong": 16, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 0, "jev_correct": 21, "local_accuracy": 0.0, "jev_accuracy": 0.21, "difference_pp": -21.0, "paired_cluster_bootstrap95_pp": [ -29.025, -13.0 ], "local_only_correct": 0, "jev_only_correct": 21, "both_correct": 0, "both_wrong": 79, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 16, "jev_correct": 51, "local_accuracy": 0.16, "jev_accuracy": 0.51, "difference_pp": -35.0, "paired_cluster_bootstrap95_pp": [ -48.0, -22.0 ], "local_only_correct": 9, "jev_only_correct": 44, "both_correct": 7, "both_wrong": 40, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 27, "jev_correct": 65, "local_accuracy": 0.27, "jev_accuracy": 0.65, "difference_pp": -38.0, "paired_cluster_bootstrap95_pp": [ -49.0, -28.0 ], "local_only_correct": 2, "jev_only_correct": 40, "both_correct": 25, "both_wrong": 33, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 24, "jev_correct": 39, "local_accuracy": 0.24, "jev_accuracy": 0.39, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -25.0, -6.0 ], "local_only_correct": 6, "jev_only_correct": 21, "both_correct": 18, "both_wrong": 55, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 8, "jev_correct": 9, "local_accuracy": 0.08, "jev_accuracy": 0.09, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -8.0, 6.0 ], "local_only_correct": 6, "jev_only_correct": 7, "both_correct": 2, "both_wrong": 85, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 29, "jev_correct": 59, "local_accuracy": 0.29, "jev_accuracy": 0.59, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -41.0, -19.0 ], "local_only_correct": 6, "jev_only_correct": 36, "both_correct": 23, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 94, "jev_correct": 91, "local_accuracy": 0.94, "jev_accuracy": 0.91, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -4.0, 10.0 ], "local_only_correct": 8, "jev_only_correct": 5, "both_correct": 86, "both_wrong": 1, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 35, "jev_correct": 88, "local_accuracy": 0.35, "jev_accuracy": 0.88, "difference_pp": -53.0, "paired_cluster_bootstrap95_pp": [ -63.0, -43.0 ], "local_only_correct": 0, "jev_only_correct": 53, "both_correct": 35, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 41, "jev_correct": 48, "local_accuracy": 0.41, "jev_accuracy": 0.48, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -17.0, 4.0 ], "local_only_correct": 11, "jev_only_correct": 18, "both_correct": 30, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 1815, "jev_correct": 2550, "local_accuracy": 0.6156716417910447, "jev_accuracy": 0.8649932157394844, "difference_pp": -24.932157394843962, "paired_cluster_bootstrap95_pp": [ -26.72998643147897, -23.168249660786973 ], "local_only_correct": 61, "jev_only_correct": 796, "both_correct": 1754, "both_wrong": 337, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 1511, "jev_correct": 2412, "local_accuracy": 0.512550881953867, "jev_accuracy": 0.8181818181818182, "difference_pp": -30.56309362279512, "paired_cluster_bootstrap95_pp": [ -32.36092265943012, -28.69742198100407 ], "local_only_correct": 100, "jev_only_correct": 1001, "both_correct": 1411, "both_wrong": 436, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 696, "jev_correct": 843, "local_accuracy": 0.5933503836317136, "jev_accuracy": 0.7186700767263428, "difference_pp": -12.531969309462914, "paired_cluster_bootstrap95_pp": [ -16.289753627213223, -8.652225087110963 ], "local_only_correct": 163, "jev_only_correct": 310, "both_correct": 533, "both_wrong": 167, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "laya": { "fresh/news": { "n": 200, "local_correct": 183, "jev_correct": 178, "local_accuracy": 0.915, "jev_accuracy": 0.89, "difference_pp": 2.5, "paired_cluster_bootstrap95_pp": [ -2.5, 8.0 ], "local_only_correct": 16, "jev_only_correct": 11, "both_correct": 167, "both_wrong": 6, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 121, "jev_correct": 120, "local_accuracy": 0.605, "jev_accuracy": 0.6, "difference_pp": 0.5, "paired_cluster_bootstrap95_pp": [ -6.0, 7.0 ], "local_only_correct": 21, "jev_only_correct": 20, "both_correct": 100, "both_wrong": 59, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 200, "jev_correct": 196, "local_accuracy": 1.0, "jev_accuracy": 0.98, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ 0.5, 4.0 ], "local_only_correct": 4, "jev_only_correct": 0, "both_correct": 196, "both_wrong": 0, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 200, "jev_correct": 179, "local_accuracy": 1.0, "jev_accuracy": 0.895, "difference_pp": 10.5, "paired_cluster_bootstrap95_pp": [ 6.5, 15.0 ], "local_only_correct": 21, "jev_only_correct": 0, "both_correct": 179, "both_wrong": 0, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 8, "jev_correct": 11, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9166666666666666, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -58.333333333333336, 8.333333333333334 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 7, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 3, "jev_correct": 10, "local_accuracy": 0.25, "jev_accuracy": 0.8333333333333334, "difference_pp": -58.333333333333336, "paired_cluster_bootstrap95_pp": [ -91.66666666666667, -16.666666666666668 ], "local_only_correct": 1, "jev_only_correct": 8, "both_correct": 2, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 8, "jev_correct": 12, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -58.333333333333336, -8.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 8, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 4, "jev_correct": 12, "local_accuracy": 0.3333333333333333, "jev_accuracy": 1.0, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -91.66666666666667, -41.666666666666664 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 4, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 97, "jev_correct": 91, "local_accuracy": 0.97, "jev_accuracy": 0.91, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ 0.0, 12.0 ], "local_only_correct": 8, "jev_only_correct": 2, "both_correct": 89, "both_wrong": 1, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 40, "jev_correct": 48, "local_accuracy": 0.4, "jev_accuracy": 0.48, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -17.0, 1.0 ], "local_only_correct": 7, "jev_only_correct": 15, "both_correct": 33, "both_wrong": 45, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 2, "jev_correct": 89, "local_accuracy": 0.02, "jev_accuracy": 0.89, "difference_pp": -87.0, "paired_cluster_bootstrap95_pp": [ -94.0, -79.0 ], "local_only_correct": 1, "jev_only_correct": 88, "both_correct": 1, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 46, "jev_correct": 48, "local_accuracy": 0.9583333333333334, "jev_accuracy": 1.0, "difference_pp": -4.166666666666666, "paired_cluster_bootstrap95_pp": [ -10.416666666666666, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 46, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/hard": { "n": 111, "local_correct": 37, "jev_correct": 81, "local_accuracy": 0.3333333333333333, "jev_accuracy": 0.7297297297297297, "difference_pp": -39.63963963963964, "paired_cluster_bootstrap95_pp": [ -51.351351351351354, -27.027027027027028 ], "local_only_correct": 10, "jev_only_correct": 54, "both_correct": 27, "both_wrong": 20, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 52, "jev_correct": 70, "local_accuracy": 0.7222222222222222, "jev_accuracy": 0.9722222222222222, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -37.5, -12.5 ], "local_only_correct": 1, "jev_only_correct": 19, "both_correct": 51, "both_wrong": 1, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 11, "jev_correct": 18, "local_accuracy": 0.5789473684210527, "jev_accuracy": 0.9473684210526315, "difference_pp": -36.84210526315789, "paired_cluster_bootstrap95_pp": [ -57.89473684210526, -15.789473684210526 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 11, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 11, "jev_correct": 20, "local_accuracy": 0.5, "jev_accuracy": 0.9090909090909091, "difference_pp": -40.909090909090914, "paired_cluster_bootstrap95_pp": [ -68.18181818181819, -13.636363636363637 ], "local_only_correct": 2, "jev_only_correct": 11, "both_correct": 9, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 7, "jev_correct": 18, "local_accuracy": 0.3888888888888889, "jev_accuracy": 1.0, "difference_pp": -61.111111111111114, "paired_cluster_bootstrap95_pp": [ -83.33333333333333, -38.888888888888886 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 7, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 14, "jev_correct": 19, "local_accuracy": 0.7368421052631579, "jev_accuracy": 1.0, "difference_pp": -26.31578947368421, "paired_cluster_bootstrap95_pp": [ -47.36842105263158, -10.394736842105981 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 14, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 7, "jev_correct": 16, "local_accuracy": 0.4117647058823529, "jev_accuracy": 0.9411764705882353, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 7, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -41.1764705882353, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5, "jev_accuracy": 0.9375, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.7222222222222222, "jev_accuracy": 1.0, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 14, "jev_correct": 21, "local_accuracy": 0.6363636363636364, "jev_accuracy": 0.9545454545454546, "difference_pp": -31.818181818181817, "paired_cluster_bootstrap95_pp": [ -50.0, -13.636363636363637 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 14, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.7333333333333333, "jev_accuracy": 1.0, "difference_pp": -26.666666666666668, "paired_cluster_bootstrap95_pp": [ -53.333333333333336, -6.666666666666667 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.7333333333333333, "jev_accuracy": 1.0, "difference_pp": -26.666666666666668, "paired_cluster_bootstrap95_pp": [ -53.333333333333336, -6.666666666666667 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7777777777777778, "jev_accuracy": 1.0, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5, "jev_accuracy": 0.9444444444444444, "difference_pp": -44.44444444444444, "paired_cluster_bootstrap95_pp": [ -72.22222222222223, -16.666666666666668 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 3, "jev_correct": 15, "local_accuracy": 0.1875, "jev_accuracy": 0.9375, "difference_pp": -75.0, "paired_cluster_bootstrap95_pp": [ -93.75, -56.25 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 3, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5, "jev_accuracy": 0.9444444444444444, "difference_pp": -44.44444444444444, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -22.22222222222222 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.75, "jev_accuracy": 0.9, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -30.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 5, "jev_correct": 18, "local_accuracy": 0.23809523809523808, "jev_accuracy": 0.8571428571428571, "difference_pp": -61.904761904761905, "paired_cluster_bootstrap95_pp": [ -85.71428571428571, -28.571428571428573 ], "local_only_correct": 2, "jev_only_correct": 15, "both_correct": 3, "both_wrong": 1, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.5625, "jev_accuracy": 1.0, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 20, "jev_correct": 23, "local_accuracy": 0.8695652173913043, "jev_accuracy": 1.0, "difference_pp": -13.043478260869565, "paired_cluster_bootstrap95_pp": [ -26.08695652173913, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 20, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 3, "jev_correct": 14, "local_accuracy": 0.21428571428571427, "jev_accuracy": 1.0, "difference_pp": -78.57142857142857, "paired_cluster_bootstrap95_pp": [ -100.0, -57.142857142857146 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 3, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 6, "jev_correct": 14, "local_accuracy": 0.375, "jev_accuracy": 0.875, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 6, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 7, "jev_correct": 14, "local_accuracy": 0.4375, "jev_accuracy": 0.875, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 7, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 11, "jev_correct": 16, "local_accuracy": 0.6875, "jev_accuracy": 1.0, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 11, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 6, "jev_correct": 16, "local_accuracy": 0.375, "jev_accuracy": 1.0, "difference_pp": -62.5, "paired_cluster_bootstrap95_pp": [ -87.5, -37.5 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 6, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 9, "jev_correct": 18, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -72.22222222222223, -27.77777777777778 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 9, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 7, "jev_correct": 17, "local_accuracy": 0.4117647058823529, "jev_accuracy": 1.0, "difference_pp": -58.82352941176471, "paired_cluster_bootstrap95_pp": [ -82.3529411764706, -35.294117647058826 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 7, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 9, "jev_correct": 18, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -72.22222222222223, -27.77777777777778 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 9, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 5, "jev_correct": 17, "local_accuracy": 0.2777777777777778, "jev_accuracy": 0.9444444444444444, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -88.88888888888889, -44.44444444444444 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 5, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 9, "jev_correct": 15, "local_accuracy": 0.5625, "jev_accuracy": 0.9375, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 9, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 5, "jev_correct": 15, "local_accuracy": 0.3333333333333333, "jev_accuracy": 1.0, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -86.66666666666667, -40.0 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 5, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.42105263157894735, "jev_accuracy": 0.8947368421052632, "difference_pp": -47.368421052631575, "paired_cluster_bootstrap95_pp": [ -68.42105263157895, -26.31578947368421 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 2, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.7222222222222222, "jev_accuracy": 1.0, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 9, "jev_correct": 19, "local_accuracy": 0.45, "jev_accuracy": 0.95, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -70.0, -30.0 ], "local_only_correct": 0, "jev_only_correct": 10, "both_correct": 9, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 15, "jev_correct": 20, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -45.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 15, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 10, "jev_correct": 16, "local_accuracy": 0.5, "jev_accuracy": 0.8, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -50.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 10, "both_wrong": 4, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 12, "jev_correct": 20, "local_accuracy": 0.5454545454545454, "jev_accuracy": 0.9090909090909091, "difference_pp": -36.36363636363637, "paired_cluster_bootstrap95_pp": [ -59.09090909090909, -9.090909090909092 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 11, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 439, "jev_correct": 629, "local_accuracy": 0.48777777777777775, "jev_accuracy": 0.6988888888888889, "difference_pp": -21.11111111111111, "paired_cluster_bootstrap95_pp": [ -26.432253698507008, -16.42503224889589 ], "local_only_correct": 68, "jev_only_correct": 258, "both_correct": 371, "both_wrong": 203, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 222, "jev_correct": 465, "local_accuracy": 0.3867595818815331, "jev_accuracy": 0.8101045296167247, "difference_pp": -42.33449477351917, "paired_cluster_bootstrap95_pp": [ -46.86411149825784, -37.97909407665505 ], "local_only_correct": 14, "jev_only_correct": 257, "both_correct": 208, "both_wrong": 95, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 113, "jev_correct": 490, "local_accuracy": 0.16142857142857142, "jev_accuracy": 0.7, "difference_pp": -53.85714285714286, "paired_cluster_bootstrap95_pp": [ -58.23699421965318, -49.56265102494243 ], "local_only_correct": 30, "jev_only_correct": 407, "both_correct": 83, "both_wrong": 180, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 7, "jev_correct": 97, "local_accuracy": 0.0175, "jev_accuracy": 0.2425, "difference_pp": -22.5, "paired_cluster_bootstrap95_pp": [ -26.75, -18.0 ], "local_only_correct": 5, "jev_only_correct": 95, "both_correct": 2, "both_wrong": 298, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 25, "jev_correct": 28, "local_accuracy": 0.8928571428571429, "jev_accuracy": 1.0, "difference_pp": -10.714285714285714, "paired_cluster_bootstrap95_pp": [ -21.428571428571427, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 25, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 24, "jev_correct": 28, "local_accuracy": 0.8571428571428571, "jev_accuracy": 1.0, "difference_pp": -14.285714285714285, "paired_cluster_bootstrap95_pp": [ -28.571428571428573, -3.5714285714285716 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 24, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 21, "jev_correct": 28, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, -10.714285714285714 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 21, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 19, "jev_correct": 28, "local_accuracy": 0.6785714285714286, "jev_accuracy": 1.0, "difference_pp": -32.142857142857146, "paired_cluster_bootstrap95_pp": [ -50.0, -14.285714285714286 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 19, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 33, "jev_correct": 40, "local_accuracy": 0.825, "jev_accuracy": 1.0, "difference_pp": -17.5, "paired_cluster_bootstrap95_pp": [ -30.0, -7.5 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 33, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 29, "jev_correct": 40, "local_accuracy": 0.725, "jev_accuracy": 1.0, "difference_pp": -27.500000000000004, "paired_cluster_bootstrap95_pp": [ -42.5, -15.0 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 29, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 22, "jev_correct": 39, "local_accuracy": 0.55, "jev_accuracy": 0.975, "difference_pp": -42.5, "paired_cluster_bootstrap95_pp": [ -57.5, -25.0 ], "local_only_correct": 1, "jev_only_correct": 18, "both_correct": 21, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 20, "jev_correct": 39, "local_accuracy": 0.5, "jev_accuracy": 0.975, "difference_pp": -47.5, "paired_cluster_bootstrap95_pp": [ -62.5, -30.0 ], "local_only_correct": 1, "jev_only_correct": 20, "both_correct": 19, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 239, "jev_correct": 237, "local_accuracy": 0.5975, "jev_accuracy": 0.5925, "difference_pp": 0.5, "paired_cluster_bootstrap95_pp": [ -4.0, 4.75 ], "local_only_correct": 45, "jev_only_correct": 43, "both_correct": 194, "both_wrong": 118, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 234, "jev_correct": 277, "local_accuracy": 0.78, "jev_accuracy": 0.9233333333333333, "difference_pp": -14.333333333333334, "paired_cluster_bootstrap95_pp": [ -19.333333333333332, -9.666666666666666 ], "local_only_correct": 7, "jev_only_correct": 50, "both_correct": 227, "both_wrong": 16, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 285, "jev_correct": 377, "local_accuracy": 0.7125, "jev_accuracy": 0.9425, "difference_pp": -23.0, "paired_cluster_bootstrap95_pp": [ -27.50625, -18.75 ], "local_only_correct": 4, "jev_only_correct": 96, "both_correct": 281, "both_wrong": 19, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 381, "jev_correct": 346, "local_accuracy": 0.9525, "jev_accuracy": 0.865, "difference_pp": 8.75, "paired_cluster_bootstrap95_pp": [ 5.49375, 12.5 ], "local_only_correct": 46, "jev_only_correct": 11, "both_correct": 335, "both_wrong": 8, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 391, "jev_correct": 360, "local_accuracy": 0.9775, "jev_accuracy": 0.9, "difference_pp": 7.75, "paired_cluster_bootstrap95_pp": [ 4.5, 11.0 ], "local_only_correct": 38, "jev_only_correct": 7, "both_correct": 353, "both_wrong": 2, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 179, "jev_correct": 225, "local_accuracy": 0.5966666666666667, "jev_accuracy": 0.75, "difference_pp": -15.333333333333332, "paired_cluster_bootstrap95_pp": [ -20.666666666666668, -9.666666666666666 ], "local_only_correct": 14, "jev_only_correct": 60, "both_correct": 165, "both_wrong": 61, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 399, "jev_correct": 393, "local_accuracy": 0.9975, "jev_accuracy": 0.9825, "difference_pp": 1.5, "paired_cluster_bootstrap95_pp": [ 0.5, 2.75 ], "local_only_correct": 6, "jev_only_correct": 0, "both_correct": 393, "both_wrong": 1, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 212, "jev_correct": 265, "local_accuracy": 0.53, "jev_accuracy": 0.6625, "difference_pp": -13.25, "paired_cluster_bootstrap95_pp": [ -17.25, -9.5 ], "local_only_correct": 11, "jev_only_correct": 64, "both_correct": 201, "both_wrong": 124, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 53, "jev_correct": 75, "local_accuracy": 0.53, "jev_accuracy": 0.75, "difference_pp": -22.0, "paired_cluster_bootstrap95_pp": [ -37.0, -7.0 ], "local_only_correct": 20, "jev_only_correct": 42, "both_correct": 33, "both_wrong": 5, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 50, "jev_correct": 56, "local_accuracy": 0.5, "jev_accuracy": 0.56, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -15.0, 3.0 ], "local_only_correct": 9, "jev_only_correct": 15, "both_correct": 41, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 48, "jev_correct": 49, "local_accuracy": 0.48, "jev_accuracy": 0.49, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -9.0, 7.024999999999864 ], "local_only_correct": 8, "jev_only_correct": 9, "both_correct": 40, "both_wrong": 43, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 24, "jev_correct": 48, "local_accuracy": 0.24, "jev_accuracy": 0.48, "difference_pp": -24.0, "paired_cluster_bootstrap95_pp": [ -34.0, -14.0 ], "local_only_correct": 4, "jev_only_correct": 28, "both_correct": 20, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 70, "jev_correct": 79, "local_accuracy": 0.7, "jev_accuracy": 0.79, "difference_pp": -9.0, "paired_cluster_bootstrap95_pp": [ -15.024999999999999, -2.0 ], "local_only_correct": 2, "jev_only_correct": 11, "both_correct": 68, "both_wrong": 19, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 1, "jev_correct": 6, "local_accuracy": 0.01, "jev_accuracy": 0.06, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -10.0, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 0, "both_wrong": 93, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 57, "jev_correct": 64, "local_accuracy": 0.57, "jev_accuracy": 0.64, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -17.0, 3.0 ], "local_only_correct": 9, "jev_only_correct": 16, "both_correct": 48, "both_wrong": 27, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 28, "jev_correct": 37, "local_accuracy": 0.28, "jev_accuracy": 0.37, "difference_pp": -9.0, "paired_cluster_bootstrap95_pp": [ -19.0, 1.0 ], "local_only_correct": 9, "jev_only_correct": 18, "both_correct": 19, "both_wrong": 54, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 6, "jev_correct": 16, "local_accuracy": 0.06, "jev_accuracy": 0.16, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -18.0, -2.0 ], "local_only_correct": 4, "jev_only_correct": 14, "both_correct": 2, "both_wrong": 80, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 4, "jev_correct": 34, "local_accuracy": 0.04, "jev_accuracy": 0.34, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -40.0, -20.0 ], "local_only_correct": 4, "jev_only_correct": 34, "both_correct": 0, "both_wrong": 62, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 79, "jev_correct": 83, "local_accuracy": 0.79, "jev_accuracy": 0.83, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -11.0, 3.0 ], "local_only_correct": 4, "jev_only_correct": 8, "both_correct": 75, "both_wrong": 13, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 8, "jev_correct": 21, "local_accuracy": 0.08, "jev_accuracy": 0.21, "difference_pp": -13.0, "paired_cluster_bootstrap95_pp": [ -23.0, -4.0 ], "local_only_correct": 7, "jev_only_correct": 20, "both_correct": 1, "both_wrong": 72, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 26, "jev_correct": 51, "local_accuracy": 0.26, "jev_accuracy": 0.51, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -37.0, -14.0 ], "local_only_correct": 7, "jev_only_correct": 32, "both_correct": 19, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 41, "jev_correct": 65, "local_accuracy": 0.41, "jev_accuracy": 0.65, "difference_pp": -24.0, "paired_cluster_bootstrap95_pp": [ -34.0, -14.0 ], "local_only_correct": 4, "jev_only_correct": 28, "both_correct": 37, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 31, "jev_correct": 39, "local_accuracy": 0.31, "jev_accuracy": 0.39, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -17.0, 1.0 ], "local_only_correct": 7, "jev_only_correct": 15, "both_correct": 24, "both_wrong": 54, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 4, "jev_correct": 9, "local_accuracy": 0.04, "jev_accuracy": 0.09, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -11.0, 1.0 ], "local_only_correct": 3, "jev_only_correct": 8, "both_correct": 1, "both_wrong": 88, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 36, "jev_correct": 59, "local_accuracy": 0.36, "jev_accuracy": 0.59, "difference_pp": -23.0, "paired_cluster_bootstrap95_pp": [ -34.0, -12.0 ], "local_only_correct": 7, "jev_only_correct": 30, "both_correct": 29, "both_wrong": 34, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 96, "jev_correct": 91, "local_accuracy": 0.96, "jev_accuracy": 0.91, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -1.0, 11.0 ], "local_only_correct": 8, "jev_only_correct": 3, "both_correct": 88, "both_wrong": 1, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 45, "jev_correct": 88, "local_accuracy": 0.45, "jev_accuracy": 0.88, "difference_pp": -43.0, "paired_cluster_bootstrap95_pp": [ -53.0, -32.0 ], "local_only_correct": 2, "jev_only_correct": 45, "both_correct": 43, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 44, "jev_correct": 48, "local_accuracy": 0.44, "jev_accuracy": 0.48, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -13.0, 5.0 ], "local_only_correct": 8, "jev_only_correct": 12, "both_correct": 36, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 1962, "jev_correct": 2550, "local_accuracy": 0.6655359565807327, "jev_accuracy": 0.8649932157394844, "difference_pp": -19.94572591587517, "paired_cluster_bootstrap95_pp": [ -21.573948439620082, -18.384497964721852 ], "local_only_correct": 77, "jev_only_correct": 665, "both_correct": 1885, "both_wrong": 321, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 621, "jev_correct": 2412, "local_accuracy": 0.21065128900949798, "jev_accuracy": 0.8181818181818182, "difference_pp": -60.75305291723202, "paired_cluster_bootstrap95_pp": [ -62.65264586160109, -58.85345997286296 ], "local_only_correct": 83, "jev_only_correct": 1874, "both_correct": 538, "both_wrong": 453, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 626, "jev_correct": 843, "local_accuracy": 0.5336743393009378, "jev_accuracy": 0.7186700767263428, "difference_pp": -18.499573742540495, "paired_cluster_bootstrap95_pp": [ -23.082678635509925, -13.616286284165065 ], "local_only_correct": 148, "jev_only_correct": 365, "both_correct": 478, "both_wrong": 182, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "nev-2b": { "fresh/news": { "n": 200, "local_correct": 177, "jev_correct": 178, "local_accuracy": 0.885, "jev_accuracy": 0.89, "difference_pp": -0.5, "paired_cluster_bootstrap95_pp": [ -3.5, 3.0 ], "local_only_correct": 5, "jev_only_correct": 6, "both_correct": 172, "both_wrong": 17, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 123, "jev_correct": 120, "local_accuracy": 0.615, "jev_accuracy": 0.6, "difference_pp": 1.5, "paired_cluster_bootstrap95_pp": [ -4.0, 7.0 ], "local_only_correct": 16, "jev_only_correct": 13, "both_correct": 107, "both_wrong": 64, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 166, "jev_correct": 196, "local_accuracy": 0.83, "jev_accuracy": 0.98, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -20.0, -10.487500000000068 ], "local_only_correct": 0, "jev_only_correct": 30, "both_correct": 166, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 136, "jev_correct": 179, "local_accuracy": 0.68, "jev_accuracy": 0.895, "difference_pp": -21.5, "paired_cluster_bootstrap95_pp": [ -27.5, -15.987500000000068 ], "local_only_correct": 0, "jev_only_correct": 43, "both_correct": 136, "both_wrong": 21, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 9, "jev_correct": 11, "local_accuracy": 0.75, "jev_accuracy": 0.9166666666666666, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 9, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 6, "jev_correct": 10, "local_accuracy": 0.5, "jev_accuracy": 0.8333333333333334, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -58.333333333333336, -8.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 6, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 88, "jev_correct": 91, "local_accuracy": 0.88, "jev_accuracy": 0.91, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -7.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 88, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 48, "jev_correct": 48, "local_accuracy": 0.48, "jev_accuracy": 0.48, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -9.0, 9.0 ], "local_only_correct": 10, "jev_only_correct": 10, "both_correct": 38, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 0, "jev_correct": 89, "local_accuracy": 0.0, "jev_accuracy": 0.89, "difference_pp": -89.0, "paired_cluster_bootstrap95_pp": [ -95.0, -82.0 ], "local_only_correct": 0, "jev_only_correct": 89, "both_correct": 0, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 75, "jev_correct": 81, "local_accuracy": 0.6756756756756757, "jev_accuracy": 0.7297297297297297, "difference_pp": -5.405405405405405, "paired_cluster_bootstrap95_pp": [ -14.414414414414415, 4.504504504504505 ], "local_only_correct": 11, "jev_only_correct": 17, "both_correct": 64, "both_wrong": 19, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 63, "jev_correct": 70, "local_accuracy": 0.875, "jev_accuracy": 0.9722222222222222, "difference_pp": -9.722222222222223, "paired_cluster_bootstrap95_pp": [ -18.055555555555557, -2.7777777777777777 ], "local_only_correct": 1, "jev_only_correct": 8, "both_correct": 62, "both_wrong": 1, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.9473684210526315, "jev_accuracy": 0.9473684210526315, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9090909090909091, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -22.727272727272727, 9.090909090909092 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 18, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7777777777777778, "jev_accuracy": 1.0, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 16, "jev_correct": 16, "local_accuracy": 0.9411764705882353, "jev_accuracy": 0.9411764705882353, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 17.647058823529413 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.7222222222222222, "jev_accuracy": 1.0, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 19, "jev_correct": 21, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9545454545454546, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -22.727272727272727, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9444444444444444, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4375, "jev_accuracy": 0.9375, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9444444444444444, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8, "jev_accuracy": 0.9, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 7, "jev_correct": 18, "local_accuracy": 0.3333333333333333, "jev_accuracy": 0.8571428571428571, "difference_pp": -52.38095238095239, "paired_cluster_bootstrap95_pp": [ -76.19047619047619, -23.80952380952381 ], "local_only_correct": 1, "jev_only_correct": 12, "both_correct": 6, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 10, "jev_correct": 14, "local_accuracy": 0.625, "jev_accuracy": 0.875, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 10, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.75, "jev_accuracy": 0.875, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -43.75, 18.75 ], "local_only_correct": 2, "jev_only_correct": 4, "both_correct": 10, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6111111111111112, "jev_accuracy": 0.9444444444444444, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.75, "jev_accuracy": 0.9375, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 7, "jev_correct": 15, "local_accuracy": 0.4666666666666667, "jev_accuracy": 1.0, "difference_pp": -53.333333333333336, "paired_cluster_bootstrap95_pp": [ -80.0, -26.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 7, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.47368421052631576, "jev_accuracy": 0.8947368421052632, "difference_pp": -42.10526315789473, "paired_cluster_bootstrap95_pp": [ -68.42105263157895, -21.05263157894737 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 2, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 6, "jev_correct": 17, "local_accuracy": 0.35294117647058826, "jev_accuracy": 1.0, "difference_pp": -64.70588235294117, "paired_cluster_bootstrap95_pp": [ -88.23529411764706, -41.1764705882353 ], "local_only_correct": 0, "jev_only_correct": 11, "both_correct": 6, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 13, "jev_correct": 19, "local_accuracy": 0.65, "jev_accuracy": 0.95, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -50.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 13, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.45, "jev_accuracy": 0.8, "difference_pp": -35.0, "paired_cluster_bootstrap95_pp": [ -55.0, -15.0 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 4, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 13.636363636363637 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 461, "jev_correct": 629, "local_accuracy": 0.5122222222222222, "jev_accuracy": 0.6988888888888889, "difference_pp": -18.666666666666668, "paired_cluster_bootstrap95_pp": [ -22.990724233157138, -14.914638195888198 ], "local_only_correct": 47, "jev_only_correct": 215, "both_correct": 414, "both_wrong": 224, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 350, "jev_correct": 465, "local_accuracy": 0.6097560975609756, "jev_accuracy": 0.8101045296167247, "difference_pp": -20.034843205574912, "paired_cluster_bootstrap95_pp": [ -24.21602787456446, -16.027874564459932 ], "local_only_correct": 23, "jev_only_correct": 138, "both_correct": 327, "both_wrong": 86, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 219, "jev_correct": 490, "local_accuracy": 0.31285714285714283, "jev_accuracy": 0.7, "difference_pp": -38.714285714285715, "paired_cluster_bootstrap95_pp": [ -42.95779706977497, -34.28072392725934 ], "local_only_correct": 34, "jev_only_correct": 305, "both_correct": 185, "both_wrong": 176, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 33, "jev_correct": 97, "local_accuracy": 0.0825, "jev_accuracy": 0.2425, "difference_pp": -16.0, "paired_cluster_bootstrap95_pp": [ -20.5, -11.25 ], "local_only_correct": 22, "jev_only_correct": 86, "both_correct": 11, "both_wrong": 281, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 35, "jev_correct": 40, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -22.5, -2.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 35, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 34, "jev_correct": 40, "local_accuracy": 0.85, "jev_accuracy": 1.0, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -27.5, -5.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 34, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 33, "jev_correct": 39, "local_accuracy": 0.825, "jev_accuracy": 0.975, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -27.5, -5.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 33, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 34, "jev_correct": 39, "local_accuracy": 0.85, "jev_accuracy": 0.975, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -25.0, -2.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 34, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 237, "jev_correct": 237, "local_accuracy": 0.5925, "jev_accuracy": 0.5925, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -3.75, 3.75 ], "local_only_correct": 30, "jev_only_correct": 30, "both_correct": 207, "both_wrong": 133, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 268, "jev_correct": 277, "local_accuracy": 0.8933333333333333, "jev_accuracy": 0.9233333333333333, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -6.333333333333333, 0.3333333333333333 ], "local_only_correct": 9, "jev_only_correct": 18, "both_correct": 259, "both_wrong": 14, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 346, "jev_correct": 377, "local_accuracy": 0.865, "jev_accuracy": 0.9425, "difference_pp": -7.75, "paired_cluster_bootstrap95_pp": [ -11.25, -4.5 ], "local_only_correct": 11, "jev_only_correct": 42, "both_correct": 335, "both_wrong": 12, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 355, "jev_correct": 346, "local_accuracy": 0.8875, "jev_accuracy": 0.865, "difference_pp": 2.25, "paired_cluster_bootstrap95_pp": [ 0.0, 4.5 ], "local_only_correct": 16, "jev_only_correct": 7, "both_correct": 339, "both_wrong": 38, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 287, "jev_correct": 360, "local_accuracy": 0.7175, "jev_accuracy": 0.9, "difference_pp": -18.25, "paired_cluster_bootstrap95_pp": [ -22.25, -14.5 ], "local_only_correct": 2, "jev_only_correct": 75, "both_correct": 285, "both_wrong": 38, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 249, "jev_correct": 225, "local_accuracy": 0.83, "jev_accuracy": 0.75, "difference_pp": 8.0, "paired_cluster_bootstrap95_pp": [ 3.3333333333333335, 12.666666666666666 ], "local_only_correct": 39, "jev_only_correct": 15, "both_correct": 210, "both_wrong": 36, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 337, "jev_correct": 393, "local_accuracy": 0.8425, "jev_accuracy": 0.9825, "difference_pp": -14.000000000000002, "paired_cluster_bootstrap95_pp": [ -17.50625, -10.5 ], "local_only_correct": 1, "jev_only_correct": 57, "both_correct": 336, "both_wrong": 6, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 267, "jev_correct": 265, "local_accuracy": 0.6675, "jev_accuracy": 0.6625, "difference_pp": 0.5, "paired_cluster_bootstrap95_pp": [ -3.75, 4.506249999999966 ], "local_only_correct": 38, "jev_only_correct": 36, "both_correct": 229, "both_wrong": 97, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 75, "jev_correct": 75, "local_accuracy": 0.75, "jev_accuracy": 0.75, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -10.0, 10.0 ], "local_only_correct": 13, "jev_only_correct": 13, "both_correct": 62, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 52, "jev_correct": 56, "local_accuracy": 0.52, "jev_accuracy": 0.56, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -10.0, 3.0 ], "local_only_correct": 4, "jev_only_correct": 8, "both_correct": 48, "both_wrong": 40, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 49, "jev_correct": 49, "local_accuracy": 0.49, "jev_accuracy": 0.49, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -8.0, 8.024999999999864 ], "local_only_correct": 9, "jev_only_correct": 9, "both_correct": 40, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 47, "jev_correct": 48, "local_accuracy": 0.47, "jev_accuracy": 0.48, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 3.0 ], "local_only_correct": 2, "jev_only_correct": 3, "both_correct": 45, "both_wrong": 50, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 80, "jev_correct": 79, "local_accuracy": 0.8, "jev_accuracy": 0.79, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 3, "both_correct": 76, "both_wrong": 17, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 7, "jev_correct": 6, "local_accuracy": 0.07, "jev_accuracy": 0.06, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -3.0, 5.0 ], "local_only_correct": 3, "jev_only_correct": 2, "both_correct": 4, "both_wrong": 91, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 60, "jev_correct": 64, "local_accuracy": 0.6, "jev_accuracy": 0.64, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -12.0, 4.0 ], "local_only_correct": 6, "jev_only_correct": 10, "both_correct": 54, "both_wrong": 30, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 43, "jev_correct": 37, "local_accuracy": 0.43, "jev_accuracy": 0.37, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ -1.0, 14.0 ], "local_only_correct": 10, "jev_only_correct": 4, "both_correct": 33, "both_wrong": 53, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 11, "jev_correct": 16, "local_accuracy": 0.11, "jev_accuracy": 0.16, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -11.0, 1.0 ], "local_only_correct": 3, "jev_only_correct": 8, "both_correct": 8, "both_wrong": 81, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 13, "jev_correct": 34, "local_accuracy": 0.13, "jev_accuracy": 0.34, "difference_pp": -21.0, "paired_cluster_bootstrap95_pp": [ -31.0, -11.0 ], "local_only_correct": 4, "jev_only_correct": 25, "both_correct": 9, "both_wrong": 62, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 86, "jev_correct": 83, "local_accuracy": 0.86, "jev_accuracy": 0.83, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -4.0, 10.0 ], "local_only_correct": 8, "jev_only_correct": 5, "both_correct": 78, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 21, "jev_correct": 21, "local_accuracy": 0.21, "jev_accuracy": 0.21, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -9.0, 9.0 ], "local_only_correct": 10, "jev_only_correct": 10, "both_correct": 11, "both_wrong": 69, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 49, "jev_correct": 51, "local_accuracy": 0.49, "jev_accuracy": 0.51, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -10.0, 7.0 ], "local_only_correct": 8, "jev_only_correct": 10, "both_correct": 41, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 0, "jev_correct": 65, "local_accuracy": 0.0, "jev_accuracy": 0.65, "difference_pp": -65.0, "paired_cluster_bootstrap95_pp": [ -75.0, -56.0 ], "local_only_correct": 0, "jev_only_correct": 65, "both_correct": 0, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 42, "jev_correct": 39, "local_accuracy": 0.42, "jev_accuracy": 0.39, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -4.0, 11.0 ], "local_only_correct": 9, "jev_only_correct": 6, "both_correct": 33, "both_wrong": 52, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 12, "jev_correct": 9, "local_accuracy": 0.12, "jev_accuracy": 0.09, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -3.0, 10.0 ], "local_only_correct": 7, "jev_only_correct": 4, "both_correct": 5, "both_wrong": 84, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 60, "jev_correct": 59, "local_accuracy": 0.6, "jev_accuracy": 0.59, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -6.0, 8.0 ], "local_only_correct": 7, "jev_only_correct": 6, "both_correct": 53, "both_wrong": 34, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 91, "jev_correct": 91, "local_accuracy": 0.91, "jev_accuracy": 0.91, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -4.0, 4.0 ], "local_only_correct": 2, "jev_only_correct": 2, "both_correct": 89, "both_wrong": 7, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 0, "jev_correct": 88, "local_accuracy": 0.0, "jev_accuracy": 0.88, "difference_pp": -88.0, "paired_cluster_bootstrap95_pp": [ -94.0, -81.0 ], "local_only_correct": 0, "jev_only_correct": 88, "both_correct": 0, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 47, "jev_correct": 48, "local_accuracy": 0.47, "jev_accuracy": 0.48, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -11.0, 9.0 ], "local_only_correct": 13, "jev_only_correct": 14, "both_correct": 34, "both_wrong": 39, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2449, "jev_correct": 2550, "local_accuracy": 0.8307327001356852, "jev_accuracy": 0.8649932157394844, "difference_pp": -3.426051560379918, "paired_cluster_bootstrap95_pp": [ -4.545454545454546, -2.4084124830393487 ], "local_only_correct": 80, "jev_only_correct": 181, "both_correct": 2369, "both_wrong": 318, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2183, "jev_correct": 2412, "local_accuracy": 0.7405020352781547, "jev_accuracy": 0.8181818181818182, "difference_pp": -7.767978290366349, "paired_cluster_bootstrap95_pp": [ -9.226594301221168, -6.343283582089552 ], "local_only_correct": 107, "jev_only_correct": 336, "both_correct": 2076, "both_wrong": 429, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 797, "jev_correct": 843, "local_accuracy": 0.6794543904518329, "jev_accuracy": 0.7186700767263428, "difference_pp": -3.9215686274509802, "paired_cluster_bootstrap95_pp": [ -6.733747544980239, -0.9327491261166293 ], "local_only_correct": 121, "jev_only_correct": 167, "both_correct": 676, "both_wrong": 209, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "nimble-9b": { "fresh/news": { "n": 200, "local_correct": 176, "jev_correct": 178, "local_accuracy": 0.88, "jev_accuracy": 0.89, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 2.0 ], "local_only_correct": 4, "jev_only_correct": 6, "both_correct": 172, "both_wrong": 18, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 118, "jev_correct": 120, "local_accuracy": 0.59, "jev_accuracy": 0.6, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -5.012499999999999, 3.5 ], "local_only_correct": 9, "jev_only_correct": 11, "both_correct": 109, "both_wrong": 71, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 165, "jev_correct": 196, "local_accuracy": 0.825, "jev_accuracy": 0.98, "difference_pp": -15.5, "paired_cluster_bootstrap95_pp": [ -21.0, -11.0 ], "local_only_correct": 0, "jev_only_correct": 31, "both_correct": 165, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 170, "jev_correct": 179, "local_accuracy": 0.85, "jev_accuracy": 0.895, "difference_pp": -4.5, "paired_cluster_bootstrap95_pp": [ -8.5, -0.5 ], "local_only_correct": 4, "jev_only_correct": 13, "both_correct": 166, "both_wrong": 17, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 10, "jev_correct": 11, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9166666666666666, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 10, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 10, "jev_correct": 10, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.8333333333333334, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 10, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 11, "jev_correct": 12, "local_accuracy": 0.9166666666666666, "jev_accuracy": 1.0, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 11, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 94, "jev_correct": 91, "local_accuracy": 0.94, "jev_accuracy": 0.91, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -3.0, 9.0 ], "local_only_correct": 6, "jev_only_correct": 3, "both_correct": 88, "both_wrong": 3, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 45, "jev_correct": 48, "local_accuracy": 0.45, "jev_accuracy": 0.48, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -11.0, 5.0 ], "local_only_correct": 7, "jev_only_correct": 10, "both_correct": 38, "both_wrong": 45, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 84, "jev_correct": 89, "local_accuracy": 0.84, "jev_accuracy": 0.89, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.0, 1.0 ], "local_only_correct": 3, "jev_only_correct": 8, "both_correct": 81, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 65, "jev_correct": 81, "local_accuracy": 0.5855855855855856, "jev_accuracy": 0.7297297297297297, "difference_pp": -14.414414414414415, "paired_cluster_bootstrap95_pp": [ -22.52252252252252, -6.306306306306307 ], "local_only_correct": 3, "jev_only_correct": 19, "both_correct": 62, "both_wrong": 27, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 68, "jev_correct": 70, "local_accuracy": 0.9444444444444444, "jev_accuracy": 0.9722222222222222, "difference_pp": -2.7777777777777777, "paired_cluster_bootstrap95_pp": [ -12.5, 4.166666666666667 ], "local_only_correct": 2, "jev_only_correct": 4, "both_correct": 66, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.8947368421052632, "jev_accuracy": 0.9473684210526315, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 18, "jev_correct": 20, "local_accuracy": 0.8181818181818182, "jev_accuracy": 0.9090909090909091, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -22.727272727272727, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 18, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 16, "jev_correct": 16, "local_accuracy": 0.9411764705882353, "jev_accuracy": 0.9411764705882353, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8666666666666667, "jev_accuracy": 1.0, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -41.1764705882353, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 21, "jev_correct": 21, "local_accuracy": 0.9545454545454546, "jev_accuracy": 0.9545454545454546, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 21, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9444444444444444, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 18, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 0.9444444444444444, "difference_pp": 5.555555555555555, "paired_cluster_bootstrap95_pp": [ 0.0, 16.666666666666668 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.9, "jev_accuracy": 0.9, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -15.0, 15.0 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 8, "jev_correct": 18, "local_accuracy": 0.38095238095238093, "jev_accuracy": 0.8571428571428571, "difference_pp": -47.61904761904761, "paired_cluster_bootstrap95_pp": [ -71.42857142857143, -23.80952380952381 ], "local_only_correct": 1, "jev_only_correct": 11, "both_correct": 7, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 15, "jev_correct": 14, "local_accuracy": 0.9375, "jev_accuracy": 0.875, "difference_pp": 6.25, "paired_cluster_bootstrap95_pp": [ 0.0, 18.75 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.6875, "jev_accuracy": 0.875, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -37.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/reading_level": { "n": 16, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.8125, "jev_accuracy": 1.0, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -43.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9444444444444444, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8421052631578947, "jev_accuracy": 0.8947368421052632, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 2, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 16, "jev_correct": 19, "local_accuracy": 0.8, "jev_accuracy": 0.95, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -35.0, 5.0 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 15, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 19, "jev_correct": 16, "local_accuracy": 0.95, "jev_accuracy": 0.8, "difference_pp": 15.0, "paired_cluster_bootstrap95_pp": [ -5.0, 35.0 ], "local_only_correct": 4, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 584, "jev_correct": 629, "local_accuracy": 0.6488888888888888, "jev_accuracy": 0.6988888888888889, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -8.708190803005046, -0.8715961493409623 ], "local_only_correct": 58, "jev_only_correct": 103, "both_correct": 526, "both_wrong": 213, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 398, "jev_correct": 465, "local_accuracy": 0.6933797909407665, "jev_accuracy": 0.8101045296167247, "difference_pp": -11.67247386759582, "paired_cluster_bootstrap95_pp": [ -15.853658536585366, -8.013937282229966 ], "local_only_correct": 33, "jev_only_correct": 100, "both_correct": 365, "both_wrong": 76, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 316, "jev_correct": 490, "local_accuracy": 0.4514285714285714, "jev_accuracy": 0.7, "difference_pp": -24.857142857142858, "paired_cluster_bootstrap95_pp": [ -28.93626165945254, -20.542082738944366 ], "local_only_correct": 44, "jev_only_correct": 218, "both_correct": 272, "both_wrong": 166, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 74, "jev_correct": 97, "local_accuracy": 0.185, "jev_accuracy": 0.2425, "difference_pp": -5.75, "paired_cluster_bootstrap95_pp": [ -10.25, -1.0 ], "local_only_correct": 32, "jev_only_correct": 55, "both_correct": 42, "both_wrong": 271, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en/reversed": { "n": 40, "local_correct": 39, "jev_correct": 40, "local_accuracy": 0.975, "jev_accuracy": 1.0, "difference_pp": -2.5, "paired_cluster_bootstrap95_pp": [ -7.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 39, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 39, "jev_correct": 40, "local_accuracy": 0.975, "jev_accuracy": 1.0, "difference_pp": -2.5, "paired_cluster_bootstrap95_pp": [ -7.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 39, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 40, "jev_correct": 39, "local_accuracy": 1.0, "jev_accuracy": 0.975, "difference_pp": 2.5, "paired_cluster_bootstrap95_pp": [ 0.0, 7.5 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 39, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 39, "jev_correct": 39, "local_accuracy": 0.975, "jev_accuracy": 0.975, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -7.5, 7.5 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 38, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 234, "jev_correct": 237, "local_accuracy": 0.585, "jev_accuracy": 0.5925, "difference_pp": -0.75, "paired_cluster_bootstrap95_pp": [ -3.5, 2.0 ], "local_only_correct": 15, "jev_only_correct": 18, "both_correct": 219, "both_wrong": 148, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 278, "jev_correct": 277, "local_accuracy": 0.9266666666666666, "jev_accuracy": 0.9233333333333333, "difference_pp": 0.33333333333333337, "paired_cluster_bootstrap95_pp": [ -3.0, 3.3333333333333335 ], "local_only_correct": 12, "jev_only_correct": 11, "both_correct": 266, "both_wrong": 11, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 342, "jev_correct": 377, "local_accuracy": 0.855, "jev_accuracy": 0.9425, "difference_pp": -8.75, "paired_cluster_bootstrap95_pp": [ -12.0, -5.5 ], "local_only_correct": 8, "jev_only_correct": 43, "both_correct": 334, "both_wrong": 15, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 354, "jev_correct": 346, "local_accuracy": 0.885, "jev_accuracy": 0.865, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -0.5, 4.25 ], "local_only_correct": 16, "jev_only_correct": 8, "both_correct": 338, "both_wrong": 38, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 337, "jev_correct": 360, "local_accuracy": 0.8425, "jev_accuracy": 0.9, "difference_pp": -5.75, "paired_cluster_bootstrap95_pp": [ -8.5, -3.25 ], "local_only_correct": 4, "jev_only_correct": 27, "both_correct": 333, "both_wrong": 36, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 240, "jev_correct": 225, "local_accuracy": 0.8, "jev_accuracy": 0.75, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ 1.6666666666666667, 8.333333333333334 ], "local_only_correct": 20, "jev_only_correct": 5, "both_correct": 220, "both_wrong": 55, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 337, "jev_correct": 393, "local_accuracy": 0.8425, "jev_accuracy": 0.9825, "difference_pp": -14.000000000000002, "paired_cluster_bootstrap95_pp": [ -17.75, -10.5 ], "local_only_correct": 2, "jev_only_correct": 58, "both_correct": 335, "both_wrong": 5, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 341, "jev_correct": 265, "local_accuracy": 0.8525, "jev_accuracy": 0.6625, "difference_pp": 19.0, "paired_cluster_bootstrap95_pp": [ 14.5, 23.5 ], "local_only_correct": 89, "jev_only_correct": 13, "both_correct": 252, "both_wrong": 46, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 73, "jev_correct": 75, "local_accuracy": 0.73, "jev_accuracy": 0.75, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -8.0, 4.0 ], "local_only_correct": 4, "jev_only_correct": 6, "both_correct": 69, "both_wrong": 21, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 52, "jev_correct": 56, "local_accuracy": 0.52, "jev_accuracy": 0.56, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -12.0, 4.0 ], "local_only_correct": 6, "jev_only_correct": 10, "both_correct": 46, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 48, "jev_correct": 49, "local_accuracy": 0.48, "jev_accuracy": 0.49, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -8.0, 7.0 ], "local_only_correct": 7, "jev_only_correct": 8, "both_correct": 41, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 43, "jev_correct": 48, "local_accuracy": 0.43, "jev_accuracy": 0.48, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -10.0, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 42, "both_wrong": 51, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 79, "jev_correct": 79, "local_accuracy": 0.79, "jev_accuracy": 0.79, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -5.0, 5.0 ], "local_only_correct": 3, "jev_only_correct": 3, "both_correct": 76, "both_wrong": 18, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 9, "jev_correct": 6, "local_accuracy": 0.09, "jev_accuracy": 0.06, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -3.0, 9.024999999999864 ], "local_only_correct": 7, "jev_only_correct": 4, "both_correct": 2, "both_wrong": 87, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 62, "jev_correct": 64, "local_accuracy": 0.62, "jev_accuracy": 0.64, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -9.0, 5.0 ], "local_only_correct": 6, "jev_only_correct": 8, "both_correct": 56, "both_wrong": 30, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 40, "jev_correct": 37, "local_accuracy": 0.4, "jev_accuracy": 0.37, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -3.0, 9.0 ], "local_only_correct": 6, "jev_only_correct": 3, "both_correct": 34, "both_wrong": 57, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 5, "jev_correct": 16, "local_accuracy": 0.05, "jev_accuracy": 0.16, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -18.0, -5.0 ], "local_only_correct": 1, "jev_only_correct": 12, "both_correct": 4, "both_wrong": 83, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 35, "jev_correct": 34, "local_accuracy": 0.35, "jev_accuracy": 0.34, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -9.0, 11.0 ], "local_only_correct": 14, "jev_only_correct": 13, "both_correct": 21, "both_wrong": 52, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 84, "jev_correct": 83, "local_accuracy": 0.84, "jev_accuracy": 0.83, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 3, "both_correct": 80, "both_wrong": 13, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 3, "jev_correct": 21, "local_accuracy": 0.03, "jev_accuracy": 0.21, "difference_pp": -18.0, "paired_cluster_bootstrap95_pp": [ -27.0, -9.0 ], "local_only_correct": 3, "jev_only_correct": 21, "both_correct": 0, "both_wrong": 76, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 43, "jev_correct": 51, "local_accuracy": 0.43, "jev_accuracy": 0.51, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -15.0, -1.9750000000001364 ], "local_only_correct": 2, "jev_only_correct": 10, "both_correct": 41, "both_wrong": 47, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 70, "jev_correct": 65, "local_accuracy": 0.7, "jev_accuracy": 0.65, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -1.0, 12.0 ], "local_only_correct": 8, "jev_only_correct": 3, "both_correct": 62, "both_wrong": 27, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 39, "jev_correct": 39, "local_accuracy": 0.39, "jev_accuracy": 0.39, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -7.0, 8.0 ], "local_only_correct": 8, "jev_only_correct": 8, "both_correct": 31, "both_wrong": 53, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 7, "jev_correct": 9, "local_accuracy": 0.07, "jev_accuracy": 0.09, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -8.0, 4.0 ], "local_only_correct": 3, "jev_only_correct": 5, "both_correct": 4, "both_wrong": 88, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 60, "jev_correct": 59, "local_accuracy": 0.6, "jev_accuracy": 0.59, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 3, "both_correct": 56, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 95, "jev_correct": 91, "local_accuracy": 0.95, "jev_accuracy": 0.91, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -2.0, 10.0 ], "local_only_correct": 7, "jev_only_correct": 3, "both_correct": 88, "both_wrong": 2, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 85, "jev_correct": 88, "local_accuracy": 0.85, "jev_accuracy": 0.88, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -8.0, 2.0 ], "local_only_correct": 2, "jev_only_correct": 5, "both_correct": 83, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 45, "jev_correct": 48, "local_accuracy": 0.45, "jev_accuracy": 0.48, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -10.0, 4.0 ], "local_only_correct": 5, "jev_only_correct": 8, "both_correct": 40, "both_wrong": 47, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2411, "jev_correct": 2550, "local_accuracy": 0.817842605156038, "jev_accuracy": 0.8649932157394844, "difference_pp": -4.715061058344641, "paired_cluster_bootstrap95_pp": [ -5.903154681139755, -3.5956580732700134 ], "local_only_correct": 84, "jev_only_correct": 223, "both_correct": 2327, "both_wrong": 314, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2181, "jev_correct": 2412, "local_accuracy": 0.7398236092265943, "jev_accuracy": 0.8181818181818182, "difference_pp": -7.835820895522389, "paired_cluster_bootstrap95_pp": [ -9.226594301221168, -6.546811397557666 ], "local_only_correct": 100, "jev_only_correct": 331, "both_correct": 2081, "both_wrong": 436, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 636, "jev_correct": 843, "local_accuracy": 0.5421994884910486, "jev_accuracy": 0.7186700767263428, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -22.002360700333988, -13.966551366576905 ], "local_only_correct": 60, "jev_only_correct": 267, "both_correct": 576, "both_wrong": 270, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "plumb-4b": { "fresh/news": { "n": 200, "local_correct": 171, "jev_correct": 178, "local_accuracy": 0.855, "jev_accuracy": 0.89, "difference_pp": -3.5000000000000004, "paired_cluster_bootstrap95_pp": [ -7.0, -0.5 ], "local_only_correct": 2, "jev_only_correct": 9, "both_correct": 169, "both_wrong": 20, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 116, "jev_correct": 120, "local_accuracy": 0.58, "jev_accuracy": 0.6, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -7.0, 3.0 ], "local_only_correct": 12, "jev_only_correct": 16, "both_correct": 104, "both_wrong": 68, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 184, "jev_correct": 196, "local_accuracy": 0.92, "jev_accuracy": 0.98, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -9.5, -3.0 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 184, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 183, "jev_correct": 179, "local_accuracy": 0.915, "jev_accuracy": 0.895, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -0.5, 5.0 ], "local_only_correct": 6, "jev_only_correct": 2, "both_correct": 177, "both_wrong": 15, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 9, "jev_correct": 11, "local_accuracy": 0.75, "jev_accuracy": 0.9166666666666666, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -41.666666666666664, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 9, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 9, "jev_correct": 10, "local_accuracy": 0.75, "jev_accuracy": 0.8333333333333334, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 9, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 89, "jev_correct": 91, "local_accuracy": 0.89, "jev_accuracy": 0.91, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -5.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 89, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 47, "jev_correct": 48, "local_accuracy": 0.47, "jev_accuracy": 0.48, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -9.0, 7.0 ], "local_only_correct": 8, "jev_only_correct": 9, "both_correct": 39, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 77, "jev_correct": 89, "local_accuracy": 0.77, "jev_accuracy": 0.89, "difference_pp": -12.0, "paired_cluster_bootstrap95_pp": [ -19.0, -5.0 ], "local_only_correct": 1, "jev_only_correct": 13, "both_correct": 76, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 89, "jev_correct": 81, "local_accuracy": 0.8018018018018018, "jev_accuracy": 0.7297297297297297, "difference_pp": 7.207207207207207, "paired_cluster_bootstrap95_pp": [ 0.9009009009009009, 14.414414414414415 ], "local_only_correct": 12, "jev_only_correct": 4, "both_correct": 77, "both_wrong": 18, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 70, "jev_correct": 70, "local_accuracy": 0.9722222222222222, "jev_accuracy": 0.9722222222222222, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -5.555555555555555, 5.555555555555555 ], "local_only_correct": 2, "jev_only_correct": 2, "both_correct": 68, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.9473684210526315, "jev_accuracy": 0.9473684210526315, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 15.789473684210526 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9090909090909091, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.9375, "jev_accuracy": 1.0, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.8235294117647058, "jev_accuracy": 0.9411764705882353, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 19, "jev_correct": 21, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9545454545454546, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -22.727272727272727, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7777777777777778, "jev_accuracy": 0.9444444444444444, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -38.888888888888886, 5.555555555555555 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 13, "jev_correct": 15, "local_accuracy": 0.8125, "jev_accuracy": 0.9375, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7777777777777778, "jev_accuracy": 0.9444444444444444, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.85, "jev_accuracy": 0.9, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 12, "jev_correct": 18, "local_accuracy": 0.5714285714285714, "jev_accuracy": 0.8571428571428571, "difference_pp": -28.57142857142857, "paired_cluster_bootstrap95_pp": [ -57.142857142857146, 0.0 ], "local_only_correct": 2, "jev_only_correct": 8, "both_correct": 10, "both_wrong": 1, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 13, "jev_correct": 14, "local_accuracy": 0.8125, "jev_accuracy": 0.875, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -25.0, 12.5 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.75, "jev_accuracy": 0.875, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -37.5, 12.5 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.875, "jev_accuracy": 1.0, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7222222222222222, "jev_accuracy": 0.9444444444444444, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.6875, "jev_accuracy": 0.9375, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.6842105263157895, "jev_accuracy": 0.8947368421052632, "difference_pp": -21.052631578947366, "paired_cluster_bootstrap95_pp": [ -42.10526315789474, 5.2631578947368425 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 13, "jev_correct": 19, "local_accuracy": 0.65, "jev_accuracy": 0.95, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -50.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 13, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 20, "jev_correct": 20, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 10, "jev_correct": 16, "local_accuracy": 0.5, "jev_accuracy": 0.8, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -50.0, -10.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 10, "both_wrong": 4, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 13.636363636363637 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 579, "jev_correct": 629, "local_accuracy": 0.6433333333333333, "jev_accuracy": 0.6988888888888889, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -10.855800355581495, -0.26811018873992604 ], "local_only_correct": 67, "jev_only_correct": 117, "both_correct": 512, "both_wrong": 204, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 428, "jev_correct": 465, "local_accuracy": 0.7456445993031359, "jev_accuracy": 0.8101045296167247, "difference_pp": -6.445993031358885, "paired_cluster_bootstrap95_pp": [ -9.930313588850174, -3.1358885017421603 ], "local_only_correct": 30, "jev_only_correct": 67, "both_correct": 398, "both_wrong": 79, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 337, "jev_correct": 490, "local_accuracy": 0.48142857142857143, "jev_accuracy": 0.7, "difference_pp": -21.857142857142858, "paired_cluster_bootstrap95_pp": [ -26.241311803806113, -17.755681818181817 ], "local_only_correct": 46, "jev_only_correct": 199, "both_correct": 291, "both_wrong": 164, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 57, "jev_correct": 97, "local_accuracy": 0.1425, "jev_accuracy": 0.2425, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -14.5, -5.75 ], "local_only_correct": 25, "jev_only_correct": 65, "both_correct": 32, "both_wrong": 278, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb": { "n": 28, "local_correct": 26, "jev_correct": 28, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -17.857142857142858, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 26, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en/reversed": { "n": 40, "local_correct": 37, "jev_correct": 40, "local_accuracy": 0.925, "jev_accuracy": 1.0, "difference_pp": -7.5, "paired_cluster_bootstrap95_pp": [ -17.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 37, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 38, "jev_correct": 40, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 38, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 34, "jev_correct": 39, "local_accuracy": 0.85, "jev_accuracy": 0.975, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -22.5, -2.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 34, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 37, "jev_correct": 39, "local_accuracy": 0.925, "jev_accuracy": 0.975, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 37, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 218, "jev_correct": 237, "local_accuracy": 0.545, "jev_accuracy": 0.5925, "difference_pp": -4.75, "paired_cluster_bootstrap95_pp": [ -8.0, -1.25 ], "local_only_correct": 19, "jev_only_correct": 38, "both_correct": 199, "both_wrong": 144, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 269, "jev_correct": 277, "local_accuracy": 0.8966666666666666, "jev_accuracy": 0.9233333333333333, "difference_pp": -2.666666666666667, "paired_cluster_bootstrap95_pp": [ -5.666666666666667, 0.3333333333333333 ], "local_only_correct": 7, "jev_only_correct": 15, "both_correct": 262, "both_wrong": 16, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 350, "jev_correct": 377, "local_accuracy": 0.875, "jev_accuracy": 0.9425, "difference_pp": -6.75, "paired_cluster_bootstrap95_pp": [ -9.75, -4.0 ], "local_only_correct": 5, "jev_only_correct": 32, "both_correct": 345, "both_wrong": 18, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 336, "jev_correct": 346, "local_accuracy": 0.84, "jev_accuracy": 0.865, "difference_pp": -2.5, "paired_cluster_bootstrap95_pp": [ -5.0, -0.25 ], "local_only_correct": 5, "jev_only_correct": 15, "both_correct": 331, "both_wrong": 49, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 364, "jev_correct": 360, "local_accuracy": 0.91, "jev_accuracy": 0.9, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -1.25, 3.25 ], "local_only_correct": 12, "jev_only_correct": 8, "both_correct": 352, "both_wrong": 28, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 227, "jev_correct": 225, "local_accuracy": 0.7566666666666667, "jev_accuracy": 0.75, "difference_pp": 0.6666666666666667, "paired_cluster_bootstrap95_pp": [ -3.0, 4.333333333333333 ], "local_only_correct": 16, "jev_only_correct": 14, "both_correct": 211, "both_wrong": 59, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 369, "jev_correct": 393, "local_accuracy": 0.9225, "jev_accuracy": 0.9825, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -8.5, -3.75 ], "local_only_correct": 0, "jev_only_correct": 24, "both_correct": 369, "both_wrong": 7, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 262, "jev_correct": 265, "local_accuracy": 0.655, "jev_accuracy": 0.6625, "difference_pp": -0.75, "paired_cluster_bootstrap95_pp": [ -4.5, 3.0 ], "local_only_correct": 27, "jev_only_correct": 30, "both_correct": 235, "both_wrong": 108, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 72, "jev_correct": 75, "local_accuracy": 0.72, "jev_accuracy": 0.75, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -9.0, 3.0 ], "local_only_correct": 3, "jev_only_correct": 6, "both_correct": 69, "both_wrong": 22, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 55, "jev_correct": 56, "local_accuracy": 0.55, "jev_accuracy": 0.56, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 6.0 ], "local_only_correct": 5, "jev_only_correct": 6, "both_correct": 50, "both_wrong": 39, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 48, "jev_correct": 49, "local_accuracy": 0.48, "jev_accuracy": 0.49, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -6.0, 4.0 ], "local_only_correct": 3, "jev_only_correct": 4, "both_correct": 45, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 42, "jev_correct": 48, "local_accuracy": 0.42, "jev_accuracy": 0.48, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -12.0, 0.0 ], "local_only_correct": 3, "jev_only_correct": 9, "both_correct": 39, "both_wrong": 49, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 76, "jev_correct": 79, "local_accuracy": 0.76, "jev_accuracy": 0.79, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -8.0, 2.0 ], "local_only_correct": 2, "jev_only_correct": 5, "both_correct": 74, "both_wrong": 19, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 7, "jev_correct": 6, "local_accuracy": 0.07, "jev_accuracy": 0.06, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -3.0, 5.0 ], "local_only_correct": 3, "jev_only_correct": 2, "both_correct": 4, "both_wrong": 91, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 66, "jev_correct": 64, "local_accuracy": 0.66, "jev_accuracy": 0.64, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -5.0, 9.0 ], "local_only_correct": 7, "jev_only_correct": 5, "both_correct": 59, "both_wrong": 29, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 42, "jev_correct": 37, "local_accuracy": 0.42, "jev_accuracy": 0.37, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -1.0, 12.0 ], "local_only_correct": 8, "jev_only_correct": 3, "both_correct": 34, "both_wrong": 55, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.09, "jev_accuracy": 0.16, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -15.0, 2.0 ], "local_only_correct": 6, "jev_only_correct": 13, "both_correct": 3, "both_wrong": 78, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 38, "jev_correct": 34, "local_accuracy": 0.38, "jev_accuracy": 0.34, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -5.0, 13.0 ], "local_only_correct": 12, "jev_only_correct": 8, "both_correct": 26, "both_wrong": 54, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 86, "jev_correct": 83, "local_accuracy": 0.86, "jev_accuracy": 0.83, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -3.0, 9.0 ], "local_only_correct": 6, "jev_only_correct": 3, "both_correct": 80, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 14, "jev_correct": 21, "local_accuracy": 0.14, "jev_accuracy": 0.21, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -16.0, 1.0 ], "local_only_correct": 7, "jev_only_correct": 14, "both_correct": 7, "both_wrong": 72, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 45, "jev_correct": 51, "local_accuracy": 0.45, "jev_accuracy": 0.51, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -15.0, 3.0 ], "local_only_correct": 7, "jev_only_correct": 13, "both_correct": 38, "both_wrong": 42, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 64, "jev_correct": 65, "local_accuracy": 0.64, "jev_accuracy": 0.65, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -7.0, 5.0 ], "local_only_correct": 4, "jev_only_correct": 5, "both_correct": 60, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 35, "jev_correct": 39, "local_accuracy": 0.35, "jev_accuracy": 0.39, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -10.0, 2.0 ], "local_only_correct": 3, "jev_only_correct": 7, "both_correct": 32, "both_wrong": 58, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 9, "jev_correct": 9, "local_accuracy": 0.09, "jev_accuracy": 0.09, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -5.0, 5.0 ], "local_only_correct": 3, "jev_only_correct": 3, "both_correct": 6, "both_wrong": 88, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 56, "jev_correct": 59, "local_accuracy": 0.56, "jev_accuracy": 0.59, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -9.0, 3.0 ], "local_only_correct": 3, "jev_only_correct": 6, "both_correct": 53, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 88, "jev_correct": 91, "local_accuracy": 0.88, "jev_accuracy": 0.91, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -7.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 88, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 74, "jev_correct": 88, "local_accuracy": 0.74, "jev_accuracy": 0.88, "difference_pp": -14.000000000000002, "paired_cluster_bootstrap95_pp": [ -21.0, -7.0 ], "local_only_correct": 1, "jev_only_correct": 15, "both_correct": 73, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 52, "jev_correct": 48, "local_accuracy": 0.52, "jev_accuracy": 0.48, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -5.0, 13.0 ], "local_only_correct": 12, "jev_only_correct": 8, "both_correct": 40, "both_wrong": 40, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2438, "jev_correct": 2550, "local_accuracy": 0.8270013568521031, "jev_accuracy": 0.8649932157394844, "difference_pp": -3.7991858887381276, "paired_cluster_bootstrap95_pp": [ -4.884667571234735, -2.7137042062415198 ], "local_only_correct": 70, "jev_only_correct": 182, "both_correct": 2368, "both_wrong": 328, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2174, "jev_correct": 2412, "local_accuracy": 0.737449118046133, "jev_accuracy": 0.8181818181818182, "difference_pp": -8.07327001356852, "paired_cluster_bootstrap95_pp": [ -9.396200814111262, -6.852103120759837 ], "local_only_correct": 77, "jev_only_correct": 315, "both_correct": 2097, "both_wrong": 459, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 806, "jev_correct": 843, "local_accuracy": 0.6871270247229326, "jev_accuracy": 0.7186700767263428, "difference_pp": -3.154305200341006, "paired_cluster_bootstrap95_pp": [ -5.649901904382588, -0.8655966674834611 ], "local_only_correct": 110, "jev_only_correct": 147, "both_correct": 696, "both_wrong": 220, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "tev1": { "fresh/news": { "n": 200, "local_correct": 175, "jev_correct": 178, "local_accuracy": 0.875, "jev_accuracy": 0.89, "difference_pp": -1.5, "paired_cluster_bootstrap95_pp": [ -5.0, 2.0 ], "local_only_correct": 4, "jev_only_correct": 7, "both_correct": 171, "both_wrong": 18, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 118, "jev_correct": 120, "local_accuracy": 0.59, "jev_accuracy": 0.6, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -5.5, 4.0 ], "local_only_correct": 12, "jev_only_correct": 14, "both_correct": 106, "both_wrong": 68, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 190, "jev_correct": 196, "local_accuracy": 0.95, "jev_accuracy": 0.98, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -5.5, -1.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 190, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 175, "jev_correct": 179, "local_accuracy": 0.875, "jev_accuracy": 0.895, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -5.0, 1.0 ], "local_only_correct": 3, "jev_only_correct": 7, "both_correct": 172, "both_wrong": 18, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 8, "jev_correct": 11, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9166666666666666, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 8, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 10, "jev_correct": 10, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.8333333333333334, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 10, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 96, "jev_correct": 91, "local_accuracy": 0.96, "jev_accuracy": 0.91, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -1.0, 11.0 ], "local_only_correct": 7, "jev_only_correct": 2, "both_correct": 89, "both_wrong": 2, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 48, "jev_correct": 48, "local_accuracy": 0.48, "jev_accuracy": 0.48, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -8.0, 8.0 ], "local_only_correct": 8, "jev_only_correct": 8, "both_correct": 40, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 0, "jev_correct": 89, "local_accuracy": 0.0, "jev_accuracy": 0.89, "difference_pp": -89.0, "paired_cluster_bootstrap95_pp": [ -95.0, -82.0 ], "local_only_correct": 0, "jev_only_correct": 89, "both_correct": 0, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 57, "jev_correct": 81, "local_accuracy": 0.5135135135135135, "jev_accuracy": 0.7297297297297297, "difference_pp": -21.62162162162162, "paired_cluster_bootstrap95_pp": [ -30.63063063063063, -13.513513513513514 ], "local_only_correct": 3, "jev_only_correct": 27, "both_correct": 54, "both_wrong": 27, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 69, "jev_correct": 70, "local_accuracy": 0.9583333333333334, "jev_accuracy": 0.9722222222222222, "difference_pp": -1.3888888888888888, "paired_cluster_bootstrap95_pp": [ -9.722222222222221, 5.555555555555555 ], "local_only_correct": 2, "jev_only_correct": 3, "both_correct": 67, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.8947368421052632, "jev_accuracy": 0.9473684210526315, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9090909090909091, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 2, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.7647058823529411, "jev_accuracy": 0.9411764705882353, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 19, "jev_correct": 21, "local_accuracy": 0.8636363636363636, "jev_accuracy": 0.9545454545454546, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -22.727272727272727, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9444444444444444, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -22.22222222222222, 11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 9, "jev_correct": 15, "local_accuracy": 0.5625, "jev_accuracy": 0.9375, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 9, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7222222222222222, "jev_accuracy": 0.9444444444444444, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 19, "jev_correct": 18, "local_accuracy": 0.95, "jev_accuracy": 0.9, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ 0.0, 15.0 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 12, "jev_correct": 18, "local_accuracy": 0.5714285714285714, "jev_accuracy": 0.8571428571428571, "difference_pp": -28.57142857142857, "paired_cluster_bootstrap95_pp": [ -52.38095238095238, -4.761904761904762 ], "local_only_correct": 1, "jev_only_correct": 7, "both_correct": 11, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.8571428571428571, "jev_accuracy": 1.0, "difference_pp": -14.285714285714285, "paired_cluster_bootstrap95_pp": [ -35.714285714285715, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 14, "jev_correct": 14, "local_accuracy": 0.875, "jev_accuracy": 0.875, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -18.75, 18.75 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 13, "jev_correct": 14, "local_accuracy": 0.8125, "jev_accuracy": 0.875, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -25.0, 12.5 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.6666666666666666, "jev_accuracy": 0.9444444444444444, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.625, "jev_accuracy": 0.9375, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 9, "jev_correct": 15, "local_accuracy": 0.6, "jev_accuracy": 1.0, "difference_pp": -40.0, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -13.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 9, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -41.1764705882353, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.6842105263157895, "jev_accuracy": 0.8947368421052632, "difference_pp": -21.052631578947366, "paired_cluster_bootstrap95_pp": [ -42.10526315789474, -5.2631578947368425 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 2, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -47.05882352941177, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 16, "jev_correct": 19, "local_accuracy": 0.8, "jev_accuracy": 0.95, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -35.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 16, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.7, "jev_accuracy": 0.8, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 4, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 13.636363636363637 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 584, "jev_correct": 629, "local_accuracy": 0.6488888888888888, "jev_accuracy": 0.6988888888888889, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -7.86546856482801, -2.433734706269947 ], "local_only_correct": 42, "jev_only_correct": 87, "both_correct": 542, "both_wrong": 229, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 432, "jev_correct": 465, "local_accuracy": 0.7526132404181185, "jev_accuracy": 0.8101045296167247, "difference_pp": -5.7491289198606275, "paired_cluster_bootstrap95_pp": [ -9.059233449477352, -2.264808362369338 ], "local_only_correct": 31, "jev_only_correct": 64, "both_correct": 401, "both_wrong": 78, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 316, "jev_correct": 490, "local_accuracy": 0.4514285714285714, "jev_accuracy": 0.7, "difference_pp": -24.857142857142858, "paired_cluster_bootstrap95_pp": [ -29.083094555873924, -20.649584964424132 ], "local_only_correct": 45, "jev_only_correct": 219, "both_correct": 271, "both_wrong": 165, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 60, "jev_correct": 97, "local_accuracy": 0.15, "jev_accuracy": 0.2425, "difference_pp": -9.25, "paired_cluster_bootstrap95_pp": [ -14.25, -4.5 ], "local_only_correct": 36, "jev_only_correct": 73, "both_correct": 24, "both_wrong": 267, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en/reversed": { "n": 40, "local_correct": 39, "jev_correct": 40, "local_accuracy": 0.975, "jev_accuracy": 1.0, "difference_pp": -2.5, "paired_cluster_bootstrap95_pp": [ -7.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 39, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 39, "jev_correct": 40, "local_accuracy": 0.975, "jev_accuracy": 1.0, "difference_pp": -2.5, "paired_cluster_bootstrap95_pp": [ -7.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 39, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 37, "jev_correct": 39, "local_accuracy": 0.925, "jev_accuracy": 0.975, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 37, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 37, "jev_correct": 39, "local_accuracy": 0.925, "jev_accuracy": 0.975, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 37, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 231, "jev_correct": 237, "local_accuracy": 0.5775, "jev_accuracy": 0.5925, "difference_pp": -1.5, "paired_cluster_bootstrap95_pp": [ -5.0, 2.25 ], "local_only_correct": 27, "jev_only_correct": 33, "both_correct": 204, "both_wrong": 136, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 268, "jev_correct": 277, "local_accuracy": 0.8933333333333333, "jev_accuracy": 0.9233333333333333, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -6.0, 0.0 ], "local_only_correct": 6, "jev_only_correct": 15, "both_correct": 262, "both_wrong": 17, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 333, "jev_correct": 377, "local_accuracy": 0.8325, "jev_accuracy": 0.9425, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -14.75, -7.25 ], "local_only_correct": 10, "jev_only_correct": 54, "both_correct": 323, "both_wrong": 13, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 352, "jev_correct": 346, "local_accuracy": 0.88, "jev_accuracy": 0.865, "difference_pp": 1.5, "paired_cluster_bootstrap95_pp": [ -1.0, 4.0 ], "local_only_correct": 16, "jev_only_correct": 10, "both_correct": 336, "both_wrong": 38, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 339, "jev_correct": 360, "local_accuracy": 0.8475, "jev_accuracy": 0.9, "difference_pp": -5.25, "paired_cluster_bootstrap95_pp": [ -7.75, -2.75 ], "local_only_correct": 4, "jev_only_correct": 25, "both_correct": 335, "both_wrong": 36, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 228, "jev_correct": 225, "local_accuracy": 0.76, "jev_accuracy": 0.75, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -2.008333333333333, 4.0 ], "local_only_correct": 13, "jev_only_correct": 10, "both_correct": 215, "both_wrong": 62, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 380, "jev_correct": 393, "local_accuracy": 0.95, "jev_accuracy": 0.9825, "difference_pp": -3.25, "paired_cluster_bootstrap95_pp": [ -5.0, -1.5 ], "local_only_correct": 0, "jev_only_correct": 13, "both_correct": 380, "both_wrong": 7, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 277, "jev_correct": 265, "local_accuracy": 0.6925, "jev_accuracy": 0.6625, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -1.5, 7.5 ], "local_only_correct": 46, "jev_only_correct": 34, "both_correct": 231, "both_wrong": 89, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 77, "jev_correct": 75, "local_accuracy": 0.77, "jev_accuracy": 0.75, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -4.0, 8.0 ], "local_only_correct": 5, "jev_only_correct": 3, "both_correct": 72, "both_wrong": 20, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 57, "jev_correct": 56, "local_accuracy": 0.57, "jev_accuracy": 0.56, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -6.0, 8.0 ], "local_only_correct": 7, "jev_only_correct": 6, "both_correct": 50, "both_wrong": 37, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 47, "jev_correct": 49, "local_accuracy": 0.47, "jev_accuracy": 0.49, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -9.0, 5.0 ], "local_only_correct": 5, "jev_only_correct": 7, "both_correct": 42, "both_wrong": 46, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 41, "jev_correct": 48, "local_accuracy": 0.41, "jev_accuracy": 0.48, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -16.0, 2.0 ], "local_only_correct": 7, "jev_only_correct": 14, "both_correct": 34, "both_wrong": 45, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 77, "jev_correct": 79, "local_accuracy": 0.77, "jev_accuracy": 0.79, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -7.0, 2.0 ], "local_only_correct": 2, "jev_only_correct": 4, "both_correct": 75, "both_wrong": 19, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 5, "jev_correct": 6, "local_accuracy": 0.05, "jev_accuracy": 0.06, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 3.0 ], "local_only_correct": 2, "jev_only_correct": 3, "both_correct": 3, "both_wrong": 92, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 64, "jev_correct": 64, "local_accuracy": 0.64, "jev_accuracy": 0.64, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -8.0, 8.0 ], "local_only_correct": 9, "jev_only_correct": 9, "both_correct": 55, "both_wrong": 27, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 40, "jev_correct": 37, "local_accuracy": 0.4, "jev_accuracy": 0.37, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -4.0, 11.0 ], "local_only_correct": 9, "jev_only_correct": 6, "both_correct": 31, "both_wrong": 54, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.09, "jev_accuracy": 0.16, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -15.0, 0.024999999999863576 ], "local_only_correct": 4, "jev_only_correct": 11, "both_correct": 5, "both_wrong": 80, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 36, "jev_correct": 34, "local_accuracy": 0.36, "jev_accuracy": 0.34, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -7.0, 11.024999999999864 ], "local_only_correct": 13, "jev_only_correct": 11, "both_correct": 23, "both_wrong": 53, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 87, "jev_correct": 83, "local_accuracy": 0.87, "jev_accuracy": 0.83, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -2.0, 10.024999999999864 ], "local_only_correct": 7, "jev_only_correct": 3, "both_correct": 80, "both_wrong": 10, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 11, "jev_correct": 21, "local_accuracy": 0.11, "jev_accuracy": 0.21, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -19.0, -2.0 ], "local_only_correct": 5, "jev_only_correct": 15, "both_correct": 6, "both_wrong": 74, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 43, "jev_correct": 51, "local_accuracy": 0.43, "jev_accuracy": 0.51, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -18.0, 2.0 ], "local_only_correct": 9, "jev_only_correct": 17, "both_correct": 34, "both_wrong": 40, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 0, "jev_correct": 65, "local_accuracy": 0.0, "jev_accuracy": 0.65, "difference_pp": -65.0, "paired_cluster_bootstrap95_pp": [ -75.0, -56.0 ], "local_only_correct": 0, "jev_only_correct": 65, "both_correct": 0, "both_wrong": 35, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 42, "jev_correct": 39, "local_accuracy": 0.42, "jev_accuracy": 0.39, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -5.0, 11.0 ], "local_only_correct": 10, "jev_only_correct": 7, "both_correct": 32, "both_wrong": 51, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 9, "jev_correct": 9, "local_accuracy": 0.09, "jev_accuracy": 0.09, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -6.0, 6.0 ], "local_only_correct": 5, "jev_only_correct": 5, "both_correct": 4, "both_wrong": 86, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 59, "jev_correct": 59, "local_accuracy": 0.59, "jev_accuracy": 0.59, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -6.0, 6.0 ], "local_only_correct": 5, "jev_only_correct": 5, "both_correct": 54, "both_wrong": 36, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 94, "jev_correct": 91, "local_accuracy": 0.94, "jev_accuracy": 0.91, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -3.0, 9.0 ], "local_only_correct": 7, "jev_only_correct": 4, "both_correct": 87, "both_wrong": 2, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 0, "jev_correct": 88, "local_accuracy": 0.0, "jev_accuracy": 0.88, "difference_pp": -88.0, "paired_cluster_bootstrap95_pp": [ -94.0, -81.0 ], "local_only_correct": 0, "jev_only_correct": 88, "both_correct": 0, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 50, "jev_correct": 48, "local_accuracy": 0.5, "jev_accuracy": 0.48, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -7.0, 11.0 ], "local_only_correct": 11, "jev_only_correct": 9, "both_correct": 39, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2421, "jev_correct": 2550, "local_accuracy": 0.8212347354138398, "jev_accuracy": 0.8649932157394844, "difference_pp": -4.375848032564451, "paired_cluster_bootstrap95_pp": [ -5.495251017639077, -3.2903663500678424 ], "local_only_correct": 77, "jev_only_correct": 206, "both_correct": 2344, "both_wrong": 321, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2143, "jev_correct": 2412, "local_accuracy": 0.7269335142469471, "jev_accuracy": 0.8181818181818182, "difference_pp": -9.12483039348711, "paired_cluster_bootstrap95_pp": [ -10.380766621438262, -7.834972862957942 ], "local_only_correct": 75, "jev_only_correct": 344, "both_correct": 2068, "both_wrong": 461, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 857, "jev_correct": 843, "local_accuracy": 0.7306052855924978, "jev_accuracy": 0.7186700767263428, "difference_pp": 1.1935208866155158, "paired_cluster_bootstrap95_pp": [ -1.6510338902466488, 3.873908450704222 ], "local_only_correct": 119, "jev_only_correct": 105, "both_correct": 738, "both_wrong": 211, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "von": { "fresh/news": { "n": 200, "local_correct": 165, "jev_correct": 178, "local_accuracy": 0.825, "jev_accuracy": 0.89, "difference_pp": -6.5, "paired_cluster_bootstrap95_pp": [ -12.5, -0.5 ], "local_only_correct": 12, "jev_only_correct": 25, "both_correct": 153, "both_wrong": 10, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 177, "jev_correct": 120, "local_accuracy": 0.885, "jev_accuracy": 0.6, "difference_pp": 28.499999999999996, "paired_cluster_bootstrap95_pp": [ 22.0, 36.0 ], "local_only_correct": 61, "jev_only_correct": 4, "both_correct": 116, "both_wrong": 19, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 125, "jev_correct": 196, "local_accuracy": 0.625, "jev_accuracy": 0.98, "difference_pp": -35.5, "paired_cluster_bootstrap95_pp": [ -42.5, -29.0 ], "local_only_correct": 1, "jev_only_correct": 72, "both_correct": 124, "both_wrong": 3, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 152, "jev_correct": 179, "local_accuracy": 0.76, "jev_accuracy": 0.895, "difference_pp": -13.5, "paired_cluster_bootstrap95_pp": [ -20.5, -6.5 ], "local_only_correct": 13, "jev_only_correct": 40, "both_correct": 139, "both_wrong": 8, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 9, "jev_correct": 11, "local_accuracy": 0.75, "jev_accuracy": 0.9166666666666666, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -50.0, 16.666666666666668 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 8, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 2, "jev_correct": 10, "local_accuracy": 0.16666666666666666, "jev_accuracy": 0.8333333333333334, "difference_pp": -66.66666666666666, "paired_cluster_bootstrap95_pp": [ -100.0, -25.0 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 1, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 9, "jev_correct": 12, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 9, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/nb": { "n": 12, "local_correct": 6, "jev_correct": 12, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 6, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/agnews": { "n": 100, "local_correct": 89, "jev_correct": 91, "local_accuracy": 0.89, "jev_accuracy": 0.91, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -10.0, 5.0 ], "local_only_correct": 6, "jev_only_correct": 8, "both_correct": 83, "both_wrong": 3, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 71, "jev_correct": 48, "local_accuracy": 0.71, "jev_accuracy": 0.48, "difference_pp": 23.0, "paired_cluster_bootstrap95_pp": [ 11.0, 36.0 ], "local_only_correct": 34, "jev_only_correct": 11, "both_correct": 37, "both_wrong": 18, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 78, "jev_correct": 89, "local_accuracy": 0.78, "jev_accuracy": 0.89, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -18.0, -4.0 ], "local_only_correct": 2, "jev_only_correct": 13, "both_correct": 76, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 50, "jev_correct": 81, "local_accuracy": 0.45045045045045046, "jev_accuracy": 0.7297297297297297, "difference_pp": -27.927927927927925, "paired_cluster_bootstrap95_pp": [ -38.73873873873874, -16.216216216216218 ], "local_only_correct": 8, "jev_only_correct": 39, "both_correct": 42, "both_wrong": 22, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 46, "jev_correct": 70, "local_accuracy": 0.6388888888888888, "jev_accuracy": 0.9722222222222222, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -47.22222222222222, -18.055555555555557 ], "local_only_correct": 2, "jev_only_correct": 26, "both_correct": 44, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7368421052631579, "jev_accuracy": 0.9473684210526315, "difference_pp": -21.052631578947366, "paired_cluster_bootstrap95_pp": [ -47.36842105263158, 0.0 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 13, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 11, "jev_correct": 16, "local_accuracy": 0.6875, "jev_accuracy": 1.0, "difference_pp": -31.25, "paired_cluster_bootstrap95_pp": [ -56.25, -12.5 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 11, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 18, "jev_correct": 20, "local_accuracy": 0.8181818181818182, "jev_accuracy": 0.9090909090909091, "difference_pp": -9.090909090909092, "paired_cluster_bootstrap95_pp": [ -27.272727272727273, 9.090909090909092 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 17, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 13, "jev_correct": 16, "local_accuracy": 0.8125, "jev_accuracy": 1.0, "difference_pp": -18.75, "paired_cluster_bootstrap95_pp": [ -43.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 13, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 15, "jev_correct": 18, "local_accuracy": 0.8333333333333334, "jev_accuracy": 1.0, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 15, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 18, "jev_correct": 19, "local_accuracy": 0.9473684210526315, "jev_accuracy": 1.0, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 18, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 11, "jev_correct": 16, "local_accuracy": 0.6470588235294118, "jev_accuracy": 0.9411764705882353, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -5.882352941176471 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.7333333333333333, "jev_accuracy": 1.0, "difference_pp": -26.666666666666668, "paired_cluster_bootstrap95_pp": [ -53.333333333333336, -6.666666666666667 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 9, "jev_correct": 15, "local_accuracy": 0.5625, "jev_accuracy": 0.9375, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -18.593750000000853 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 9, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 12, "jev_correct": 18, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 12, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 15, "jev_correct": 21, "local_accuracy": 0.6818181818181818, "jev_accuracy": 0.9545454545454546, "difference_pp": -27.27272727272727, "paired_cluster_bootstrap95_pp": [ -45.45454545454545, -9.090909090909092 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 15, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 17, "jev_correct": 19, "local_accuracy": 0.8947368421052632, "jev_accuracy": 1.0, "difference_pp": -10.526315789473683, "paired_cluster_bootstrap95_pp": [ -26.31578947368421, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 17, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 10, "jev_correct": 17, "local_accuracy": 0.5882352941176471, "jev_accuracy": 1.0, "difference_pp": -41.17647058823529, "paired_cluster_bootstrap95_pp": [ -64.70588235294117, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 10, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.7333333333333333, "jev_accuracy": 1.0, "difference_pp": -26.666666666666668, "paired_cluster_bootstrap95_pp": [ -53.333333333333336, -6.666666666666667 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.7333333333333333, "jev_accuracy": 1.0, "difference_pp": -26.666666666666668, "paired_cluster_bootstrap95_pp": [ -53.333333333333336, -6.666666666666667 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 11, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9444444444444444, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5, "jev_accuracy": 0.9375, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -24.843750000000853 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.8888888888888888, "jev_accuracy": 0.9444444444444444, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 14, "jev_correct": 18, "local_accuracy": 0.7, "jev_accuracy": 0.9, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -45.0, 0.0 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 13, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.8095238095238095, "jev_accuracy": 0.8571428571428571, "difference_pp": -4.761904761904762, "paired_cluster_bootstrap95_pp": [ -19.047619047619047, 9.523809523809524 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -43.75, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 22, "jev_correct": 23, "local_accuracy": 0.9565217391304348, "jev_accuracy": 1.0, "difference_pp": -4.3478260869565215, "paired_cluster_bootstrap95_pp": [ -13.043478260869565, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 22, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 10, "jev_correct": 14, "local_accuracy": 0.7142857142857143, "jev_accuracy": 1.0, "difference_pp": -28.57142857142857, "paired_cluster_bootstrap95_pp": [ -50.0, -7.142857142857143 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 10, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 8, "jev_correct": 14, "local_accuracy": 0.5, "jev_accuracy": 0.875, "difference_pp": -37.5, "paired_cluster_bootstrap95_pp": [ -62.5, -12.5 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 8, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.75, "jev_accuracy": 0.875, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -37.5, 12.5 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 8, "jev_correct": 16, "local_accuracy": 0.5, "jev_accuracy": 1.0, "difference_pp": -50.0, "paired_cluster_bootstrap95_pp": [ -75.0, -25.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 8, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.7222222222222222, "jev_accuracy": 1.0, "difference_pp": -27.77777777777778, "paired_cluster_bootstrap95_pp": [ -50.0, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 13, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 12, "jev_correct": 18, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 12, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5, "jev_accuracy": 0.9444444444444444, "difference_pp": -44.44444444444444, "paired_cluster_bootstrap95_pp": [ -66.66666666666667, -22.22222222222222 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 11, "jev_correct": 15, "local_accuracy": 0.6875, "jev_accuracy": 0.9375, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, 0.0 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 10, "jev_correct": 15, "local_accuracy": 0.6666666666666666, "jev_accuracy": 1.0, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -60.0, -13.333333333333334 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 10, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.8235294117647058, "jev_accuracy": 1.0, "difference_pp": -17.647058823529413, "paired_cluster_bootstrap95_pp": [ -35.294117647058826, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 8, "jev_correct": 17, "local_accuracy": 0.47058823529411764, "jev_accuracy": 1.0, "difference_pp": -52.94117647058824, "paired_cluster_bootstrap95_pp": [ -76.47058823529412, -29.41176470588235 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 8, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 19, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 0.8947368421052632, "difference_pp": 10.526315789473683, "paired_cluster_bootstrap95_pp": [ 0.0, 26.31578947368421 ], "local_only_correct": 2, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6470588235294118, "jev_accuracy": 1.0, "difference_pp": -35.294117647058826, "paired_cluster_bootstrap95_pp": [ -58.8235294117647, -17.647058823529413 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 11, "jev_correct": 18, "local_accuracy": 0.6111111111111112, "jev_accuracy": 1.0, "difference_pp": -38.88888888888889, "paired_cluster_bootstrap95_pp": [ -61.111111111111114, -16.666666666666668 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 11, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 11, "jev_correct": 19, "local_accuracy": 0.55, "jev_accuracy": 0.95, "difference_pp": -40.0, "paired_cluster_bootstrap95_pp": [ -65.0, -15.0 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 10, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 18, "jev_correct": 20, "local_accuracy": 0.9, "jev_accuracy": 1.0, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 18, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.45, "jev_accuracy": 0.8, "difference_pp": -35.0, "paired_cluster_bootstrap95_pp": [ -55.0, -15.0 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 4, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 16, "jev_correct": 20, "local_accuracy": 0.7272727272727273, "jev_accuracy": 0.9090909090909091, "difference_pp": -18.181818181818183, "paired_cluster_bootstrap95_pp": [ -40.90909090909091, 0.0 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 15, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 389, "jev_correct": 629, "local_accuracy": 0.43222222222222223, "jev_accuracy": 0.6988888888888889, "difference_pp": -26.666666666666668, "paired_cluster_bootstrap95_pp": [ -33.6893012280567, -20.132480986950142 ], "local_only_correct": 105, "jev_only_correct": 345, "both_correct": 284, "both_wrong": 166, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 229, "jev_correct": 465, "local_accuracy": 0.3989547038327526, "jev_accuracy": 0.8101045296167247, "difference_pp": -41.11498257839721, "paired_cluster_bootstrap95_pp": [ -45.818815331010455, -36.7595818815331 ], "local_only_correct": 19, "jev_only_correct": 255, "both_correct": 210, "both_wrong": 90, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 141, "jev_correct": 490, "local_accuracy": 0.20142857142857143, "jev_accuracy": 0.7, "difference_pp": -49.857142857142854, "paired_cluster_bootstrap95_pp": [ -54.25099820874794, -45.55842019892632 ], "local_only_correct": 36, "jev_only_correct": 385, "both_correct": 105, "both_wrong": 174, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 5, "jev_correct": 97, "local_accuracy": 0.0125, "jev_accuracy": 0.2425, "difference_pp": -23.0, "paired_cluster_bootstrap95_pp": [ -27.25, -18.75 ], "local_only_correct": 4, "jev_only_correct": 96, "both_correct": 1, "both_wrong": 299, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 26, "jev_correct": 28, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -17.857142857142858, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 26, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en/reversed": { "n": 28, "local_correct": 26, "jev_correct": 28, "local_accuracy": 0.9285714285714286, "jev_accuracy": 1.0, "difference_pp": -7.142857142857142, "paired_cluster_bootstrap95_pp": [ -17.857142857142858, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 26, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 12, "jev_correct": 28, "local_accuracy": 0.42857142857142855, "jev_accuracy": 1.0, "difference_pp": -57.14285714285714, "paired_cluster_bootstrap95_pp": [ -75.0, -39.285714285714285 ], "local_only_correct": 0, "jev_only_correct": 16, "both_correct": 12, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 12, "jev_correct": 28, "local_accuracy": 0.42857142857142855, "jev_accuracy": 1.0, "difference_pp": -57.14285714285714, "paired_cluster_bootstrap95_pp": [ -75.0, -39.285714285714285 ], "local_only_correct": 0, "jev_only_correct": 16, "both_correct": 12, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en/reversed": { "n": 40, "local_correct": 34, "jev_correct": 40, "local_accuracy": 0.85, "jev_accuracy": 1.0, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -25.0, -5.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 34, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 34, "jev_correct": 40, "local_accuracy": 0.85, "jev_accuracy": 1.0, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -25.0, -5.0 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 34, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 15, "jev_correct": 39, "local_accuracy": 0.375, "jev_accuracy": 0.975, "difference_pp": -60.0, "paired_cluster_bootstrap95_pp": [ -75.0, -42.5 ], "local_only_correct": 1, "jev_only_correct": 25, "both_correct": 14, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 15, "jev_correct": 39, "local_accuracy": 0.375, "jev_accuracy": 0.975, "difference_pp": -60.0, "paired_cluster_bootstrap95_pp": [ -75.0, -42.5 ], "local_only_correct": 1, "jev_only_correct": 25, "both_correct": 14, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 355, "jev_correct": 237, "local_accuracy": 0.8875, "jev_accuracy": 0.5925, "difference_pp": 29.5, "paired_cluster_bootstrap95_pp": [ 25.0, 34.25 ], "local_only_correct": 124, "jev_only_correct": 6, "both_correct": 231, "both_wrong": 39, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 237, "jev_correct": 277, "local_accuracy": 0.79, "jev_accuracy": 0.9233333333333333, "difference_pp": -13.333333333333334, "paired_cluster_bootstrap95_pp": [ -17.666666666666668, -9.0 ], "local_only_correct": 4, "jev_only_correct": 44, "both_correct": 233, "both_wrong": 19, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 292, "jev_correct": 377, "local_accuracy": 0.73, "jev_accuracy": 0.9425, "difference_pp": -21.25, "paired_cluster_bootstrap95_pp": [ -26.0, -16.75 ], "local_only_correct": 9, "jev_only_correct": 94, "both_correct": 283, "both_wrong": 14, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 346, "jev_correct": 346, "local_accuracy": 0.865, "jev_accuracy": 0.865, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -3.5, 3.75 ], "local_only_correct": 27, "jev_only_correct": 27, "both_correct": 319, "both_wrong": 27, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 278, "jev_correct": 360, "local_accuracy": 0.695, "jev_accuracy": 0.9, "difference_pp": -20.5, "paired_cluster_bootstrap95_pp": [ -25.5, -15.25 ], "local_only_correct": 25, "jev_only_correct": 107, "both_correct": 253, "both_wrong": 15, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 161, "jev_correct": 225, "local_accuracy": 0.5366666666666666, "jev_accuracy": 0.75, "difference_pp": -21.333333333333336, "paired_cluster_bootstrap95_pp": [ -28.0, -14.991666666666712 ], "local_only_correct": 24, "jev_only_correct": 88, "both_correct": 137, "both_wrong": 51, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 235, "jev_correct": 393, "local_accuracy": 0.5875, "jev_accuracy": 0.9825, "difference_pp": -39.5, "paired_cluster_bootstrap95_pp": [ -44.75, -34.5 ], "local_only_correct": 3, "jev_only_correct": 161, "both_correct": 232, "both_wrong": 4, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 274, "jev_correct": 265, "local_accuracy": 0.685, "jev_accuracy": 0.6625, "difference_pp": 2.25, "paired_cluster_bootstrap95_pp": [ -4.0, 8.25 ], "local_only_correct": 88, "jev_only_correct": 79, "both_correct": 186, "both_wrong": 47, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 57, "jev_correct": 75, "local_accuracy": 0.57, "jev_accuracy": 0.75, "difference_pp": -18.0, "paired_cluster_bootstrap95_pp": [ -30.0, -5.0 ], "local_only_correct": 14, "jev_only_correct": 32, "both_correct": 43, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 34, "jev_correct": 56, "local_accuracy": 0.34, "jev_accuracy": 0.56, "difference_pp": -22.0, "paired_cluster_bootstrap95_pp": [ -31.0, -13.0 ], "local_only_correct": 3, "jev_only_correct": 25, "both_correct": 31, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 34, "jev_correct": 49, "local_accuracy": 0.34, "jev_accuracy": 0.49, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -28.0, -2.9750000000001364 ], "local_only_correct": 15, "jev_only_correct": 30, "both_correct": 19, "both_wrong": 36, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 17, "jev_correct": 48, "local_accuracy": 0.17, "jev_accuracy": 0.48, "difference_pp": -31.0, "paired_cluster_bootstrap95_pp": [ -41.0, -21.0 ], "local_only_correct": 3, "jev_only_correct": 34, "both_correct": 14, "both_wrong": 49, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 48, "jev_correct": 79, "local_accuracy": 0.48, "jev_accuracy": 0.79, "difference_pp": -31.0, "paired_cluster_bootstrap95_pp": [ -40.0, -21.0 ], "local_only_correct": 1, "jev_only_correct": 32, "both_correct": 47, "both_wrong": 20, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 1, "jev_correct": 6, "local_accuracy": 0.01, "jev_accuracy": 0.06, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -10.0, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 0, "both_wrong": 93, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 34, "jev_correct": 64, "local_accuracy": 0.34, "jev_accuracy": 0.64, "difference_pp": -30.0, "paired_cluster_bootstrap95_pp": [ -41.0, -20.0 ], "local_only_correct": 5, "jev_only_correct": 35, "both_correct": 29, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 25, "jev_correct": 37, "local_accuracy": 0.25, "jev_accuracy": 0.37, "difference_pp": -12.0, "paired_cluster_bootstrap95_pp": [ -23.0, -1.0 ], "local_only_correct": 12, "jev_only_correct": 24, "both_correct": 13, "both_wrong": 51, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 0, "jev_correct": 16, "local_accuracy": 0.0, "jev_accuracy": 0.16, "difference_pp": -16.0, "paired_cluster_bootstrap95_pp": [ -24.0, -9.0 ], "local_only_correct": 0, "jev_only_correct": 16, "both_correct": 0, "both_wrong": 84, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 13, "jev_correct": 34, "local_accuracy": 0.13, "jev_accuracy": 0.34, "difference_pp": -21.0, "paired_cluster_bootstrap95_pp": [ -30.0, -11.0 ], "local_only_correct": 4, "jev_only_correct": 25, "both_correct": 9, "both_wrong": 62, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 68, "jev_correct": 83, "local_accuracy": 0.68, "jev_accuracy": 0.83, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -23.0, -7.0 ], "local_only_correct": 2, "jev_only_correct": 17, "both_correct": 66, "both_wrong": 15, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 0, "jev_correct": 21, "local_accuracy": 0.0, "jev_accuracy": 0.21, "difference_pp": -21.0, "paired_cluster_bootstrap95_pp": [ -29.025, -13.0 ], "local_only_correct": 0, "jev_only_correct": 21, "both_correct": 0, "both_wrong": 79, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 20, "jev_correct": 51, "local_accuracy": 0.2, "jev_accuracy": 0.51, "difference_pp": -31.0, "paired_cluster_bootstrap95_pp": [ -42.0, -19.975000000000136 ], "local_only_correct": 6, "jev_only_correct": 37, "both_correct": 14, "both_wrong": 43, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 34, "jev_correct": 65, "local_accuracy": 0.34, "jev_accuracy": 0.65, "difference_pp": -31.0, "paired_cluster_bootstrap95_pp": [ -41.0, -20.0 ], "local_only_correct": 4, "jev_only_correct": 35, "both_correct": 30, "both_wrong": 31, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 21, "jev_correct": 39, "local_accuracy": 0.21, "jev_accuracy": 0.39, "difference_pp": -18.0, "paired_cluster_bootstrap95_pp": [ -29.0, -8.0 ], "local_only_correct": 9, "jev_only_correct": 27, "both_correct": 12, "both_wrong": 52, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 0, "jev_correct": 9, "local_accuracy": 0.0, "jev_accuracy": 0.09, "difference_pp": -9.0, "paired_cluster_bootstrap95_pp": [ -15.0, -4.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 0, "both_wrong": 91, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 34, "jev_correct": 59, "local_accuracy": 0.34, "jev_accuracy": 0.59, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -36.025, -14.0 ], "local_only_correct": 7, "jev_only_correct": 32, "both_correct": 27, "both_wrong": 34, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 89, "jev_correct": 91, "local_accuracy": 0.89, "jev_accuracy": 0.91, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -9.0, 5.0 ], "local_only_correct": 6, "jev_only_correct": 8, "both_correct": 83, "both_wrong": 3, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 74, "jev_correct": 88, "local_accuracy": 0.74, "jev_accuracy": 0.88, "difference_pp": -14.000000000000002, "paired_cluster_bootstrap95_pp": [ -22.0, -6.0 ], "local_only_correct": 3, "jev_only_correct": 17, "both_correct": 71, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 76, "jev_correct": 48, "local_accuracy": 0.76, "jev_accuracy": 0.48, "difference_pp": 28.000000000000004, "paired_cluster_bootstrap95_pp": [ 17.0, 39.0 ], "local_only_correct": 32, "jev_only_correct": 4, "both_correct": 44, "both_wrong": 20, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 1665, "jev_correct": 2550, "local_accuracy": 0.5647896879240163, "jev_accuracy": 0.8649932157394844, "difference_pp": -30.020352781546812, "paired_cluster_bootstrap95_pp": [ -31.886024423337854, -28.08683853459973 ], "local_only_correct": 65, "jev_only_correct": 950, "both_correct": 1600, "both_wrong": 333, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 394, "jev_correct": 2412, "local_accuracy": 0.13364993215739485, "jev_accuracy": 0.8181818181818182, "difference_pp": -68.45318860244234, "paired_cluster_bootstrap95_pp": [ -70.21709633649932, -66.58751696065129 ], "local_only_correct": 79, "jev_only_correct": 2097, "both_correct": 315, "both_wrong": 457, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 582, "jev_correct": 843, "local_accuracy": 0.4961636828644501, "jev_accuracy": 0.7186700767263428, "difference_pp": -22.25063938618926, "paired_cluster_bootstrap95_pp": [ -28.22368914254424, -16.47324921091626 ], "local_only_correct": 196, "jev_only_correct": 457, "both_correct": 386, "both_wrong": 134, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "wald-4b-v12": { "fresh/news": { "n": 200, "local_correct": 170, "jev_correct": 178, "local_accuracy": 0.85, "jev_accuracy": 0.89, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -7.0, -1.0 ], "local_only_correct": 1, "jev_only_correct": 9, "both_correct": 169, "both_wrong": 21, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 121, "jev_correct": 120, "local_accuracy": 0.605, "jev_accuracy": 0.6, "difference_pp": 0.5, "paired_cluster_bootstrap95_pp": [ -4.0, 5.0 ], "local_only_correct": 11, "jev_only_correct": 10, "both_correct": 110, "both_wrong": 69, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 184, "jev_correct": 196, "local_accuracy": 0.92, "jev_accuracy": 0.98, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -9.5, -3.0 ], "local_only_correct": 0, "jev_only_correct": 12, "both_correct": 184, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 181, "jev_correct": 179, "local_accuracy": 0.905, "jev_accuracy": 0.895, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -2.5, 5.0 ], "local_only_correct": 8, "jev_only_correct": 6, "both_correct": 173, "both_wrong": 13, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 10, "jev_correct": 11, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9166666666666666, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 10, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/nb": { "n": 12, "local_correct": 9, "jev_correct": 10, "local_accuracy": 0.75, "jev_accuracy": 0.8333333333333334, "difference_pp": -8.333333333333332, "paired_cluster_bootstrap95_pp": [ -25.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 9, "both_wrong": 2, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 83, "jev_correct": 91, "local_accuracy": 0.83, "jev_accuracy": 0.91, "difference_pp": -8.0, "paired_cluster_bootstrap95_pp": [ -14.0, -3.0 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 83, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 43, "jev_correct": 48, "local_accuracy": 0.43, "jev_accuracy": 0.48, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -13.0, 3.0 ], "local_only_correct": 6, "jev_only_correct": 11, "both_correct": 37, "both_wrong": 46, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 84, "jev_correct": 89, "local_accuracy": 0.84, "jev_accuracy": 0.89, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.0, 1.0 ], "local_only_correct": 2, "jev_only_correct": 7, "both_correct": 82, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 77, "jev_correct": 81, "local_accuracy": 0.6936936936936937, "jev_accuracy": 0.7297297297297297, "difference_pp": -3.6036036036036037, "paired_cluster_bootstrap95_pp": [ -12.612612612612613, 5.405405405405405 ], "local_only_correct": 12, "jev_only_correct": 16, "both_correct": 65, "both_wrong": 18, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 72, "jev_correct": 70, "local_accuracy": 1.0, "jev_accuracy": 0.9722222222222222, "difference_pp": 2.7777777777777777, "paired_cluster_bootstrap95_pp": [ 0.0, 6.944444444444445 ], "local_only_correct": 2, "jev_only_correct": 0, "both_correct": 70, "both_wrong": 0, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.9473684210526315, "jev_accuracy": 0.9473684210526315, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 15.789473684210526 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 21, "jev_correct": 20, "local_accuracy": 0.9545454545454546, "jev_accuracy": 0.9090909090909091, "difference_pp": 4.545454545454546, "paired_cluster_bootstrap95_pp": [ -9.090909090909092, 22.727272727272727 ], "local_only_correct": 2, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.9444444444444444, "jev_accuracy": 1.0, "difference_pp": -5.555555555555555, "paired_cluster_bootstrap95_pp": [ -16.666666666666668, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 15, "jev_correct": 16, "local_accuracy": 0.8823529411764706, "jev_accuracy": 0.9411764705882353, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -23.529411764705884, 11.764705882352942 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.875, "jev_accuracy": 0.9375, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_type": { "n": 22, "local_correct": 20, "jev_correct": 21, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9545454545454546, "difference_pp": -4.545454545454546, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 20, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8823529411764706, "jev_accuracy": 1.0, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 15, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 12, "jev_correct": 15, "local_accuracy": 0.8, "jev_accuracy": 1.0, "difference_pp": -20.0, "paired_cluster_bootstrap95_pp": [ -40.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 12, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/expense_category": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 15, "jev_correct": 17, "local_accuracy": 0.8333333333333334, "jev_accuracy": 0.9444444444444444, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 11.11111111111111 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 8, "jev_correct": 15, "local_accuracy": 0.5, "jev_accuracy": 0.9375, "difference_pp": -43.75, "paired_cluster_bootstrap95_pp": [ -68.75, -18.75 ], "local_only_correct": 0, "jev_only_correct": 7, "both_correct": 8, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7777777777777778, "jev_accuracy": 0.9444444444444444, "difference_pp": -16.666666666666664, "paired_cluster_bootstrap95_pp": [ -33.333333333333336, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 19, "jev_correct": 18, "local_accuracy": 0.95, "jev_accuracy": 0.9, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ 0.0, 15.0 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 13, "jev_correct": 18, "local_accuracy": 0.6190476190476191, "jev_accuracy": 0.8571428571428571, "difference_pp": -23.809523809523807, "paired_cluster_bootstrap95_pp": [ -47.61904761904762, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 12, "both_wrong": 2, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 11, "jev_correct": 14, "local_accuracy": 0.7857142857142857, "jev_accuracy": 1.0, "difference_pp": -21.428571428571427, "paired_cluster_bootstrap95_pp": [ -42.857142857142854, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 14, "jev_correct": 14, "local_accuracy": 0.875, "jev_accuracy": 0.875, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -18.75, 18.75 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 13, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 13, "jev_correct": 14, "local_accuracy": 0.8125, "jev_accuracy": 0.875, "difference_pp": -6.25, "paired_cluster_bootstrap95_pp": [ -18.75, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 13, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7647058823529411, "jev_accuracy": 1.0, "difference_pp": -23.52941176470588, "paired_cluster_bootstrap95_pp": [ -41.1764705882353, -5.882352941176471 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/news_topic": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 12, "jev_correct": 16, "local_accuracy": 0.75, "jev_accuracy": 1.0, "difference_pp": -25.0, "paired_cluster_bootstrap95_pp": [ -50.0, -6.25 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 12, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 16, "jev_correct": 18, "local_accuracy": 0.8888888888888888, "jev_accuracy": 1.0, "difference_pp": -11.11111111111111, "paired_cluster_bootstrap95_pp": [ -27.77777777777778, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 16, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 9, "jev_correct": 17, "local_accuracy": 0.5294117647058824, "jev_accuracy": 1.0, "difference_pp": -47.05882352941176, "paired_cluster_bootstrap95_pp": [ -70.58823529411765, -23.529411764705884 ], "local_only_correct": 0, "jev_only_correct": 8, "both_correct": 9, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 11, "jev_correct": 17, "local_accuracy": 0.6111111111111112, "jev_accuracy": 0.9444444444444444, "difference_pp": -33.33333333333333, "paired_cluster_bootstrap95_pp": [ -55.55555555555556, -11.11111111111111 ], "local_only_correct": 0, "jev_only_correct": 6, "both_correct": 11, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 15, "jev_correct": 15, "local_accuracy": 0.9375, "jev_accuracy": 0.9375, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -18.75, 18.75 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 14, "jev_correct": 17, "local_accuracy": 0.7368421052631579, "jev_accuracy": 0.8947368421052632, "difference_pp": -15.789473684210526, "paired_cluster_bootstrap95_pp": [ -31.57894736842105, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 14, "both_wrong": 2, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 12, "jev_correct": 17, "local_accuracy": 0.7058823529411765, "jev_accuracy": 1.0, "difference_pp": -29.411764705882355, "paired_cluster_bootstrap95_pp": [ -52.94117647058823, -11.764705882352942 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 16, "jev_correct": 19, "local_accuracy": 0.8, "jev_accuracy": 0.95, "difference_pp": -15.0, "paired_cluster_bootstrap95_pp": [ -35.0, 5.0 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 15, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 19, "jev_correct": 20, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 17, "jev_correct": 16, "local_accuracy": 0.85, "jev_accuracy": 0.8, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -10.0, 20.0 ], "local_only_correct": 2, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 20, "jev_correct": 20, "local_accuracy": 0.9090909090909091, "jev_accuracy": 0.9090909090909091, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -13.636363636363637, 13.636363636363637 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 19, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 595, "jev_correct": 629, "local_accuracy": 0.6611111111111111, "jev_accuracy": 0.6988888888888889, "difference_pp": -3.7777777777777777, "paired_cluster_bootstrap95_pp": [ -7.400449745626813, 0.0 ], "local_only_correct": 76, "jev_only_correct": 110, "both_correct": 519, "both_wrong": 195, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 403, "jev_correct": 465, "local_accuracy": 0.7020905923344948, "jev_accuracy": 0.8101045296167247, "difference_pp": -10.801393728222997, "paired_cluster_bootstrap95_pp": [ -14.290069686411151, -7.491289198606272 ], "local_only_correct": 23, "jev_only_correct": 85, "both_correct": 380, "both_wrong": 86, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 363, "jev_correct": 490, "local_accuracy": 0.5185714285714286, "jev_accuracy": 0.7, "difference_pp": -18.142857142857142, "paired_cluster_bootstrap95_pp": [ -22.063409369999626, -14.1200230216304 ], "local_only_correct": 51, "jev_only_correct": 178, "both_correct": 312, "both_wrong": 159, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 86, "jev_correct": 97, "local_accuracy": 0.215, "jev_accuracy": 0.2425, "difference_pp": -2.75, "paired_cluster_bootstrap95_pp": [ -7.75, 2.25 ], "local_only_correct": 49, "jev_only_correct": 60, "both_correct": 37, "both_wrong": 254, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en/reversed": { "n": 40, "local_correct": 36, "jev_correct": 40, "local_accuracy": 0.9, "jev_accuracy": 1.0, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -20.0, -2.5 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 36, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/en": { "n": 40, "local_correct": 38, "jev_correct": 40, "local_accuracy": 0.95, "jev_accuracy": 1.0, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -12.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 38, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 35, "jev_correct": 39, "local_accuracy": 0.875, "jev_accuracy": 0.975, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -20.0, -2.5 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 35, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 36, "jev_correct": 39, "local_accuracy": 0.9, "jev_accuracy": 0.975, "difference_pp": -7.5, "paired_cluster_bootstrap95_pp": [ -17.5, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 36, "both_wrong": 1, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 239, "jev_correct": 237, "local_accuracy": 0.5975, "jev_accuracy": 0.5925, "difference_pp": 0.5, "paired_cluster_bootstrap95_pp": [ -3.25, 4.0 ], "local_only_correct": 29, "jev_only_correct": 27, "both_correct": 210, "both_wrong": 134, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 276, "jev_correct": 277, "local_accuracy": 0.92, "jev_accuracy": 0.9233333333333333, "difference_pp": -0.33333333333333337, "paired_cluster_bootstrap95_pp": [ -3.0, 2.3333333333333335 ], "local_only_correct": 8, "jev_only_correct": 9, "both_correct": 268, "both_wrong": 15, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 349, "jev_correct": 377, "local_accuracy": 0.8725, "jev_accuracy": 0.9425, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -9.75, -4.5 ], "local_only_correct": 2, "jev_only_correct": 30, "both_correct": 347, "both_wrong": 21, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 338, "jev_correct": 346, "local_accuracy": 0.845, "jev_accuracy": 0.865, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -4.5, 0.25 ], "local_only_correct": 6, "jev_only_correct": 14, "both_correct": 332, "both_wrong": 48, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 358, "jev_correct": 360, "local_accuracy": 0.895, "jev_accuracy": 0.9, "difference_pp": -0.5, "paired_cluster_bootstrap95_pp": [ -2.5, 1.75 ], "local_only_correct": 8, "jev_only_correct": 10, "both_correct": 350, "both_wrong": 32, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 233, "jev_correct": 225, "local_accuracy": 0.7766666666666666, "jev_accuracy": 0.75, "difference_pp": 2.666666666666667, "paired_cluster_bootstrap95_pp": [ -0.3333333333333333, 5.666666666666667 ], "local_only_correct": 14, "jev_only_correct": 6, "both_correct": 219, "both_wrong": 61, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 363, "jev_correct": 393, "local_accuracy": 0.9075, "jev_accuracy": 0.9825, "difference_pp": -7.5, "paired_cluster_bootstrap95_pp": [ -10.25, -5.0 ], "local_only_correct": 1, "jev_only_correct": 31, "both_correct": 362, "both_wrong": 6, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 278, "jev_correct": 265, "local_accuracy": 0.695, "jev_accuracy": 0.6625, "difference_pp": 3.25, "paired_cluster_bootstrap95_pp": [ -0.5, 6.75 ], "local_only_correct": 35, "jev_only_correct": 22, "both_correct": 243, "both_wrong": 100, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 79, "jev_correct": 75, "local_accuracy": 0.79, "jev_accuracy": 0.75, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -4.0, 12.0 ], "local_only_correct": 10, "jev_only_correct": 6, "both_correct": 69, "both_wrong": 15, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 50, "jev_correct": 56, "local_accuracy": 0.5, "jev_accuracy": 0.56, "difference_pp": -6.0, "paired_cluster_bootstrap95_pp": [ -14.0, 2.0 ], "local_only_correct": 6, "jev_only_correct": 12, "both_correct": 44, "both_wrong": 38, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 50, "jev_correct": 49, "local_accuracy": 0.5, "jev_accuracy": 0.49, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -3.0, 6.0 ], "local_only_correct": 3, "jev_only_correct": 2, "both_correct": 47, "both_wrong": 48, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 53, "jev_correct": 48, "local_accuracy": 0.53, "jev_accuracy": 0.48, "difference_pp": 5.0, "paired_cluster_bootstrap95_pp": [ -1.0, 11.0 ], "local_only_correct": 8, "jev_only_correct": 3, "both_correct": 45, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 80, "jev_correct": 79, "local_accuracy": 0.8, "jev_accuracy": 0.79, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -2.0, 4.0 ], "local_only_correct": 2, "jev_only_correct": 1, "both_correct": 78, "both_wrong": 19, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 5, "jev_correct": 6, "local_accuracy": 0.05, "jev_accuracy": 0.06, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 2.0 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 4, "both_wrong": 93, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 63, "jev_correct": 64, "local_accuracy": 0.63, "jev_accuracy": 0.64, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -10.0, 8.0 ], "local_only_correct": 9, "jev_only_correct": 10, "both_correct": 54, "both_wrong": 27, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 43, "jev_correct": 37, "local_accuracy": 0.43, "jev_accuracy": 0.37, "difference_pp": 6.0, "paired_cluster_bootstrap95_pp": [ -2.0, 14.0 ], "local_only_correct": 11, "jev_only_correct": 5, "both_correct": 32, "both_wrong": 52, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 9, "jev_correct": 16, "local_accuracy": 0.09, "jev_accuracy": 0.16, "difference_pp": -7.000000000000001, "paired_cluster_bootstrap95_pp": [ -15.0, 0.0 ], "local_only_correct": 4, "jev_only_correct": 11, "both_correct": 5, "both_wrong": 80, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 20, "jev_correct": 34, "local_accuracy": 0.2, "jev_accuracy": 0.34, "difference_pp": -14.000000000000002, "paired_cluster_bootstrap95_pp": [ -23.025, -5.0 ], "local_only_correct": 5, "jev_only_correct": 19, "both_correct": 15, "both_wrong": 61, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 78, "jev_correct": 83, "local_accuracy": 0.78, "jev_accuracy": 0.83, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -10.024999999999999, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 77, "both_wrong": 16, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 5, "jev_correct": 21, "local_accuracy": 0.05, "jev_accuracy": 0.21, "difference_pp": -16.0, "paired_cluster_bootstrap95_pp": [ -24.0, -9.0 ], "local_only_correct": 0, "jev_only_correct": 16, "both_correct": 5, "both_wrong": 79, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 40, "jev_correct": 51, "local_accuracy": 0.4, "jev_accuracy": 0.51, "difference_pp": -11.0, "paired_cluster_bootstrap95_pp": [ -20.0, -3.0 ], "local_only_correct": 5, "jev_only_correct": 16, "both_correct": 35, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 69, "jev_correct": 65, "local_accuracy": 0.69, "jev_accuracy": 0.65, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -4.0, 11.0 ], "local_only_correct": 9, "jev_only_correct": 5, "both_correct": 60, "both_wrong": 26, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 41, "jev_correct": 39, "local_accuracy": 0.41, "jev_accuracy": 0.39, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -5.0, 10.0 ], "local_only_correct": 8, "jev_only_correct": 6, "both_correct": 33, "both_wrong": 53, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 9, "jev_correct": 9, "local_accuracy": 0.09, "jev_accuracy": 0.09, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -5.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 4, "both_correct": 5, "both_wrong": 87, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 60, "jev_correct": 59, "local_accuracy": 0.6, "jev_accuracy": 0.59, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 7.0 ], "local_only_correct": 5, "jev_only_correct": 4, "both_correct": 55, "both_wrong": 36, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 86, "jev_correct": 91, "local_accuracy": 0.86, "jev_accuracy": 0.91, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -11.0, 0.0 ], "local_only_correct": 1, "jev_only_correct": 6, "both_correct": 85, "both_wrong": 8, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 87, "jev_correct": 88, "local_accuracy": 0.87, "jev_accuracy": 0.88, "difference_pp": -1.0, "paired_cluster_bootstrap95_pp": [ -6.0, 4.0 ], "local_only_correct": 3, "jev_only_correct": 4, "both_correct": 84, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 43, "jev_correct": 48, "local_accuracy": 0.43, "jev_accuracy": 0.48, "difference_pp": -5.0, "paired_cluster_bootstrap95_pp": [ -14.0, 4.0 ], "local_only_correct": 9, "jev_only_correct": 14, "both_correct": 34, "both_wrong": 43, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2507, "jev_correct": 2550, "local_accuracy": 0.8504070556309362, "jev_accuracy": 0.8649932157394844, "difference_pp": -1.4586160108548167, "paired_cluster_bootstrap95_pp": [ -2.4423337856173677, -0.44097693351424694 ], "local_only_correct": 84, "jev_only_correct": 127, "both_correct": 2423, "both_wrong": 314, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2379, "jev_correct": 2412, "local_accuracy": 0.8069877883310719, "jev_accuracy": 0.8181818181818182, "difference_pp": -1.1194029850746268, "paired_cluster_bootstrap95_pp": [ -2.341417910447761, 0.10261194029850283 ], "local_only_correct": 155, "jev_only_correct": 188, "both_correct": 2224, "both_wrong": 381, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 854, "jev_correct": 843, "local_accuracy": 0.7280477408354646, "jev_accuracy": 0.7186700767263428, "difference_pp": 0.937766410912191, "paired_cluster_bootstrap95_pp": [ -2.792571579364032, 4.513748921413802 ], "local_only_correct": 163, "jev_only_correct": 152, "both_correct": 691, "both_wrong": 167, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } }, "winnow-12b-q8": { "fresh/news": { "n": 200, "local_correct": 170, "jev_correct": 178, "local_accuracy": 0.85, "jev_accuracy": 0.89, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -7.5, -1.0 ], "local_only_correct": 2, "jev_only_correct": 10, "both_correct": 168, "both_wrong": 20, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/emotion": { "n": 200, "local_correct": 120, "jev_correct": 120, "local_accuracy": 0.6, "jev_accuracy": 0.6, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -4.5, 4.5 ], "local_only_correct": 10, "jev_only_correct": 10, "both_correct": 110, "both_wrong": 70, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/spam": { "n": 200, "local_correct": 187, "jev_correct": 196, "local_accuracy": 0.935, "jev_accuracy": 0.98, "difference_pp": -4.5, "paired_cluster_bootstrap95_pp": [ -7.5, -2.0 ], "local_only_correct": 0, "jev_only_correct": 9, "both_correct": 187, "both_wrong": 4, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh/phishing": { "n": 200, "local_correct": 182, "jev_correct": 179, "local_accuracy": 0.91, "jev_accuracy": 0.895, "difference_pp": 1.5, "paired_cluster_bootstrap95_pp": [ -1.5, 5.0 ], "local_only_correct": 7, "jev_only_correct": 4, "both_correct": 175, "both_wrong": 14, "independent_groups": 200, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/routing/en": { "n": 12, "local_correct": 11, "jev_correct": 11, "local_accuracy": 0.9166666666666666, "jev_accuracy": 0.9166666666666666, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 11, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/routing/nb": { "n": 12, "local_correct": 11, "jev_correct": 10, "local_accuracy": 0.9166666666666666, "jev_accuracy": 0.8333333333333334, "difference_pp": 8.333333333333332, "paired_cluster_bootstrap95_pp": [ 0.0, 25.0 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 10, "both_wrong": 1, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fresh_business/documents/en": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "fresh_business/documents/nb": { "n": 12, "local_correct": 12, "jev_correct": 12, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 12, "both_wrong": 0, "independent_groups": 12, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jev_verified/agnews": { "n": 100, "local_correct": 88, "jev_correct": 91, "local_accuracy": 0.88, "jev_accuracy": 0.91, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -7.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 3, "both_correct": 88, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/emotiondair": { "n": 100, "local_correct": 52, "jev_correct": 48, "local_accuracy": 0.52, "jev_accuracy": 0.48, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -2.0249999999999986, 11.0 ], "local_only_correct": 8, "jev_only_correct": 4, "both_correct": 44, "both_wrong": 44, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_verified/banking77": { "n": 100, "local_correct": 0, "jev_correct": 89, "local_accuracy": 0.0, "jev_accuracy": 0.89, "difference_pp": -89.0, "paired_cluster_bootstrap95_pp": [ -95.0, -82.0 ], "local_only_correct": 0, "jev_only_correct": 89, "both_correct": 0, "both_wrong": 11, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/easy": { "n": 48, "local_correct": 48, "jev_correct": 48, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 48, "both_wrong": 0, "independent_groups": 48, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jevbench_public/hard": { "n": 111, "local_correct": 82, "jev_correct": 81, "local_accuracy": 0.7387387387387387, "jev_accuracy": 0.7297297297297297, "difference_pp": 0.9009009009009009, "paired_cluster_bootstrap95_pp": [ -7.207207207207207, 9.00900900900901 ], "local_only_correct": 10, "jev_only_correct": 9, "both_correct": 72, "both_wrong": 20, "independent_groups": 111, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jevbench_public/original": { "n": 72, "local_correct": 69, "jev_correct": 70, "local_accuracy": 0.9583333333333334, "jev_accuracy": 0.9722222222222222, "difference_pp": -1.3888888888888888, "paired_cluster_bootstrap95_pp": [ -8.333333333333334, 4.166666666666667 ], "local_only_correct": 1, "jev_only_correct": 2, "both_correct": 68, "both_wrong": 1, "independent_groups": 36, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/action_item_assignment": { "n": 19, "local_correct": 17, "jev_correct": 18, "local_accuracy": 0.8947368421052632, "jev_accuracy": 0.9473684210526315, "difference_pp": -5.263157894736842, "paired_cluster_bootstrap95_pp": [ -15.789473684210526, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/ad_policy_violation": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/allergen_present": { "n": 22, "local_correct": 22, "jev_correct": 20, "local_accuracy": 1.0, "jev_accuracy": 0.9090909090909091, "difference_pp": 9.090909090909092, "paired_cluster_bootstrap95_pp": [ 0.0, 22.727272727272727 ], "local_only_correct": 2, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/app_review_intent": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/churn_risk": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/city_service_request": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/code_review_intent": { "n": 17, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.8235294117647058, "jev_accuracy": 0.9411764705882353, "difference_pp": -11.76470588235294, "paired_cluster_bootstrap95_pp": [ -29.41176470588235, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 14, "both_wrong": 1, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/commit_intent": { "n": 15, "local_correct": 14, "jev_correct": 15, "local_accuracy": 0.9333333333333333, "jev_accuracy": 1.0, "difference_pp": -6.666666666666667, "paired_cluster_bootstrap95_pp": [ -20.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_pii": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/contains_spoiler": { "n": 16, "local_correct": 16, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 0.9375, "difference_pp": 6.25, "paired_cluster_bootstrap95_pp": [ 0.0, 18.75 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/content_moderation": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/content_type": { "n": 22, "local_correct": 21, "jev_correct": 21, "local_accuracy": 0.9545454545454546, "jev_accuracy": 0.9545454545454546, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 21, "both_wrong": 1, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/contract_clause_type": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/delivery_exception": { "n": 19, "local_correct": 19, "jev_correct": 19, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 19, "both_wrong": 0, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/dietary_vegan": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/document_type": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/email_intent_v2": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/expense_category": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/fair_housing_violation": { "n": 18, "local_correct": 18, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 0.9444444444444444, "difference_pp": 5.555555555555555, "paired_cluster_bootstrap95_pp": [ 0.0, 16.666666666666668 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/formality_level": { "n": 16, "local_correct": 15, "jev_correct": 15, "local_accuracy": 0.9375, "jev_accuracy": 0.9375, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -18.75, 18.75 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/frustration_level_v2": { "n": 18, "local_correct": 18, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 0.9444444444444444, "difference_pp": 5.555555555555555, "paired_cluster_bootstrap95_pp": [ 0.0, 16.666666666666668 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/gaming_report_type": { "n": 20, "local_correct": 18, "jev_correct": 18, "local_accuracy": 0.9, "jev_accuracy": 0.9, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 2, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/grammar_issue": { "n": 21, "local_correct": 20, "jev_correct": 18, "local_accuracy": 0.9523809523809523, "jev_accuracy": 0.8571428571428571, "difference_pp": 9.523809523809524, "paired_cluster_bootstrap95_pp": [ -9.523809523809524, 28.571428571428573 ], "local_only_correct": 3, "jev_only_correct": 1, "both_correct": 17, "both_wrong": 0, "independent_groups": 21, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/hazmat_shipping": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/home_service_routing": { "n": 23, "local_correct": 23, "jev_correct": 23, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 23, "both_wrong": 0, "independent_groups": 23, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/incident_severity_v2": { "n": 14, "local_correct": 14, "jev_correct": 14, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 14, "both_wrong": 0, "independent_groups": 14, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/insurance_claim_priority": { "n": 16, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.75, "jev_accuracy": 0.875, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -31.25, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 12, "both_wrong": 2, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/lead_qualification": { "n": 16, "local_correct": 12, "jev_correct": 14, "local_accuracy": 0.75, "jev_accuracy": 0.875, "difference_pp": -12.5, "paired_cluster_bootstrap95_pp": [ -37.5, 12.5 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 11, "both_wrong": 1, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/meeting_conflict": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/news_topic": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/oncall_route": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/phishing_email": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/question_duplicate": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/reading_level": { "n": 16, "local_correct": 16, "jev_correct": 16, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 16, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/recipe_cuisine": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/refund_eligible_v2": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/return_reason": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/review_sentiment_v2": { "n": 18, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.7222222222222222, "jev_accuracy": 0.9444444444444444, "difference_pp": -22.22222222222222, "paired_cluster_bootstrap95_pp": [ -44.44444444444444, -5.555555555555555 ], "local_only_correct": 0, "jev_only_correct": 4, "both_correct": 13, "both_wrong": 1, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/secret_leak_v2": { "n": 16, "local_correct": 15, "jev_correct": 15, "local_accuracy": 0.9375, "jev_accuracy": 0.9375, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ -18.75, 18.75 ], "local_only_correct": 1, "jev_only_correct": 1, "both_correct": 14, "both_wrong": 0, "independent_groups": 16, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/sql_injection_risk": { "n": 15, "local_correct": 15, "jev_correct": 15, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 15, "both_wrong": 0, "independent_groups": 15, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/support_department_v2": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/suspicious_transaction": { "n": 17, "local_correct": 17, "jev_correct": 17, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 17, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/symptom_triage": { "n": 19, "local_correct": 13, "jev_correct": 17, "local_accuracy": 0.6842105263157895, "jev_accuracy": 0.8947368421052632, "difference_pp": -21.052631578947366, "paired_cluster_bootstrap95_pp": [ -42.10526315789474, 5.2631578947368425 ], "local_only_correct": 1, "jev_only_correct": 5, "both_correct": 12, "both_wrong": 1, "independent_groups": 19, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/travel_policy_violation": { "n": 17, "local_correct": 16, "jev_correct": 17, "local_accuracy": 0.9411764705882353, "jev_accuracy": 1.0, "difference_pp": -5.88235294117647, "paired_cluster_bootstrap95_pp": [ -17.647058823529413, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 16, "both_wrong": 0, "independent_groups": 17, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/urgency_v2": { "n": 18, "local_correct": 18, "jev_correct": 18, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 18, "both_wrong": 0, "independent_groups": 18, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/veterinary_triage": { "n": 20, "local_correct": 17, "jev_correct": 19, "local_accuracy": 0.85, "jev_accuracy": 0.95, "difference_pp": -10.0, "paired_cluster_bootstrap95_pp": [ -30.0, 10.0 ], "local_only_correct": 1, "jev_only_correct": 3, "both_correct": 16, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/voice_assistant_intent": { "n": 20, "local_correct": 20, "jev_correct": 20, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 0, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "jabr_v2/warranty_claim_eligible": { "n": 20, "local_correct": 18, "jev_correct": 16, "local_accuracy": 0.9, "jev_accuracy": 0.8, "difference_pp": 10.0, "paired_cluster_bootstrap95_pp": [ -10.0, 30.0 ], "local_only_correct": 3, "jev_only_correct": 1, "both_correct": 15, "both_wrong": 1, "independent_groups": 20, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jabr_v2/weather_alert_severity": { "n": 22, "local_correct": 22, "jev_correct": 20, "local_accuracy": 1.0, "jev_accuracy": 0.9090909090909091, "difference_pp": 9.090909090909092, "paired_cluster_bootstrap95_pp": [ 0.0, 22.727272727272727 ], "local_only_correct": 2, "jev_only_correct": 0, "both_correct": 20, "both_wrong": 0, "independent_groups": 22, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/devtools-v1": { "n": 900, "local_correct": 610, "jev_correct": 629, "local_accuracy": 0.6777777777777778, "jev_accuracy": 0.6988888888888889, "difference_pp": -2.111111111111111, "paired_cluster_bootstrap95_pp": [ -6.189454635277945, 1.372857142857135 ], "local_only_correct": 54, "jev_only_correct": 73, "both_correct": 556, "both_wrong": 217, "independent_groups": 596, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/documents-v1": { "n": 574, "local_correct": 461, "jev_correct": 465, "local_accuracy": 0.8031358885017421, "jev_accuracy": 0.8101045296167247, "difference_pp": -0.6968641114982579, "paired_cluster_bootstrap95_pp": [ -3.658536585365854, 1.916376306620209 ], "local_only_correct": 30, "jev_only_correct": 34, "both_correct": 431, "both_wrong": 79, "independent_groups": 574, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "kev_claim/hard-v1": { "n": 700, "local_correct": 428, "jev_correct": 490, "local_accuracy": 0.6114285714285714, "jev_accuracy": 0.7, "difference_pp": -8.857142857142856, "paired_cluster_bootstrap95_pp": [ -12.750716332378223, -5.043021099937528 ], "local_only_correct": 65, "jev_only_correct": 127, "both_correct": 363, "both_wrong": 145, "independent_groups": 650, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "intern_claim/typed_decisions-test": { "n": 400, "local_correct": 87, "jev_correct": 97, "local_accuracy": 0.2175, "jev_accuracy": 0.2425, "difference_pp": -2.5, "paired_cluster_bootstrap95_pp": [ -7.75, 2.5 ], "local_only_correct": 46, "jev_only_correct": 56, "both_correct": 41, "both_wrong": 257, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/en": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/en/reversed": { "n": 28, "local_correct": 27, "jev_correct": 28, "local_accuracy": 0.9642857142857143, "jev_accuracy": 1.0, "difference_pp": -3.571428571428571, "paired_cluster_bootstrap95_pp": [ -10.714285714285714, 0.0 ], "local_only_correct": 0, "jev_only_correct": 1, "both_correct": 27, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/documents/nb": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/documents/nb/reversed": { "n": 28, "local_correct": 28, "jev_correct": 28, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 28, "both_wrong": 0, "independent_groups": 28, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en/reversed": { "n": 40, "local_correct": 40, "jev_correct": 40, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 40, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/en": { "n": 40, "local_correct": 40, "jev_correct": 40, "local_accuracy": 1.0, "jev_accuracy": 1.0, "difference_pp": 0.0, "paired_cluster_bootstrap95_pp": [ 0.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 0, "both_correct": 40, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": true, "inference_warning": "A degenerate empirical bootstrap is not proof of equivalence or zero uncertainty beyond these cases." }, "industry/routing/nb/reversed": { "n": 40, "local_correct": 40, "jev_correct": 39, "local_accuracy": 1.0, "jev_accuracy": 0.975, "difference_pp": 2.5, "paired_cluster_bootstrap95_pp": [ 0.0, 7.5 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 39, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "industry/routing/nb": { "n": 40, "local_correct": 40, "jev_correct": 39, "local_accuracy": 1.0, "jev_accuracy": 0.975, "difference_pp": 2.5, "paired_cluster_bootstrap95_pp": [ 0.0, 7.5 ], "local_only_correct": 1, "jev_only_correct": 0, "both_correct": 39, "both_wrong": 0, "independent_groups": 40, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/emotion.replication": { "n": 400, "local_correct": 246, "jev_correct": 237, "local_accuracy": 0.615, "jev_accuracy": 0.5925, "difference_pp": 2.25, "paired_cluster_bootstrap95_pp": [ -1.0, 5.5 ], "local_only_correct": 27, "jev_only_correct": 18, "both_correct": 219, "both_wrong": 136, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/intent": { "n": 300, "local_correct": 271, "jev_correct": 277, "local_accuracy": 0.9033333333333333, "jev_accuracy": 0.9233333333333333, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -4.0, 0.0 ], "local_only_correct": 2, "jev_only_correct": 8, "both_correct": 269, "both_wrong": 21, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/jailbreaking": { "n": 400, "local_correct": 378, "jev_correct": 377, "local_accuracy": 0.945, "jev_accuracy": 0.9425, "difference_pp": 0.25, "paired_cluster_bootstrap95_pp": [ -2.0, 2.5 ], "local_only_correct": 10, "jev_only_correct": 9, "both_correct": 368, "both_wrong": 13, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/news.replication": { "n": 400, "local_correct": 337, "jev_correct": 346, "local_accuracy": 0.8425, "jev_accuracy": 0.865, "difference_pp": -2.25, "paired_cluster_bootstrap95_pp": [ -4.25, -0.5 ], "local_only_correct": 3, "jev_only_correct": 12, "both_correct": 334, "both_wrong": 51, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/phishing.replication": { "n": 400, "local_correct": 353, "jev_correct": 360, "local_accuracy": 0.8825, "jev_accuracy": 0.9, "difference_pp": -1.7500000000000002, "paired_cluster_bootstrap95_pp": [ -4.0, 0.25 ], "local_only_correct": 7, "jev_only_correct": 14, "both_correct": 346, "both_wrong": 33, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/scenario": { "n": 300, "local_correct": 224, "jev_correct": 225, "local_accuracy": 0.7466666666666667, "jev_accuracy": 0.75, "difference_pp": -0.33333333333333337, "paired_cluster_bootstrap95_pp": [ -3.0, 2.3333333333333335 ], "local_only_correct": 9, "jev_only_correct": 10, "both_correct": 215, "both_wrong": 66, "independent_groups": 300, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/spam.replication": { "n": 400, "local_correct": 388, "jev_correct": 393, "local_accuracy": 0.97, "jev_accuracy": 0.9825, "difference_pp": -1.25, "paired_cluster_bootstrap95_pp": [ -2.25, -0.25 ], "local_only_correct": 0, "jev_only_correct": 5, "both_correct": 388, "both_wrong": 7, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "laya_claim/toxicity": { "n": 400, "local_correct": 283, "jev_correct": 265, "local_accuracy": 0.7075, "jev_accuracy": 0.6625, "difference_pp": 4.5, "paired_cluster_bootstrap95_pp": [ 1.0, 8.25 ], "local_only_correct": 36, "jev_only_correct": 18, "both_correct": 247, "both_wrong": 99, "independent_groups": 400, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/agent_handoff": { "n": 100, "local_correct": 78, "jev_correct": 75, "local_accuracy": 0.78, "jev_accuracy": 0.75, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -2.0, 9.0 ], "local_only_correct": 6, "jev_only_correct": 3, "both_correct": 72, "both_wrong": 19, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/banking_intent": { "n": 100, "local_correct": 47, "jev_correct": 56, "local_accuracy": 0.47, "jev_accuracy": 0.56, "difference_pp": -9.0, "paired_cluster_bootstrap95_pp": [ -16.0, -2.0 ], "local_only_correct": 3, "jev_only_correct": 12, "both_correct": 44, "both_wrong": 41, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/benefits_request": { "n": 100, "local_correct": 53, "jev_correct": 49, "local_accuracy": 0.53, "jev_accuracy": 0.49, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ 0.0, 9.0 ], "local_only_correct": 5, "jev_only_correct": 1, "both_correct": 48, "both_wrong": 46, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/clinic_request": { "n": 100, "local_correct": 49, "jev_correct": 48, "local_accuracy": 0.49, "jev_accuracy": 0.48, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -5.0, 7.0 ], "local_only_correct": 5, "jev_only_correct": 4, "both_correct": 44, "both_wrong": 47, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/document_type": { "n": 100, "local_correct": 76, "jev_correct": 79, "local_accuracy": 0.76, "jev_accuracy": 0.79, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -9.0, 2.0 ], "local_only_correct": 2, "jev_only_correct": 5, "both_correct": 74, "both_wrong": 19, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/email_triage": { "n": 100, "local_correct": 8, "jev_correct": 6, "local_accuracy": 0.08, "jev_accuracy": 0.06, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -2.0, 7.0 ], "local_only_correct": 4, "jev_only_correct": 2, "both_correct": 4, "both_wrong": 90, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/news_topic": { "n": 100, "local_correct": 67, "jev_correct": 64, "local_accuracy": 0.67, "jev_accuracy": 0.64, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -3.0, 9.0 ], "local_only_correct": 7, "jev_only_correct": 4, "both_correct": 60, "both_wrong": 29, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/paper_field": { "n": 100, "local_correct": 47, "jev_correct": 37, "local_accuracy": 0.47, "jev_accuracy": 0.37, "difference_pp": 10.0, "paired_cluster_bootstrap95_pp": [ 4.0, 17.0 ], "local_only_correct": 11, "jev_only_correct": 1, "both_correct": 36, "both_wrong": 52, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/product_feedback": { "n": 100, "local_correct": 14, "jev_correct": 16, "local_accuracy": 0.14, "jev_accuracy": 0.16, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -9.0, 5.0 ], "local_only_correct": 5, "jev_only_correct": 7, "both_correct": 9, "both_wrong": 79, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/restaurant_review": { "n": 100, "local_correct": 37, "jev_correct": 34, "local_accuracy": 0.37, "jev_accuracy": 0.34, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -6.0, 12.0 ], "local_only_correct": 12, "jev_only_correct": 9, "both_correct": 25, "both_wrong": 54, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/review_sentiment": { "n": 100, "local_correct": 84, "jev_correct": 83, "local_accuracy": 0.84, "jev_accuracy": 0.83, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -2.0, 5.0 ], "local_only_correct": 2, "jev_only_correct": 1, "both_correct": 82, "both_wrong": 15, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/screen_tags": { "n": 100, "local_correct": 19, "jev_correct": 21, "local_accuracy": 0.19, "jev_accuracy": 0.21, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -10.0, 6.0 ], "local_only_correct": 7, "jev_only_correct": 9, "both_correct": 12, "both_wrong": 72, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/sports_recap": { "n": 100, "local_correct": 52, "jev_correct": 51, "local_accuracy": 0.52, "jev_accuracy": 0.51, "difference_pp": 1.0, "paired_cluster_bootstrap95_pp": [ -4.0, 6.0 ], "local_only_correct": 4, "jev_only_correct": 3, "both_correct": 48, "both_wrong": 45, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_intent": { "n": 100, "local_correct": 61, "jev_correct": 65, "local_accuracy": 0.61, "jev_accuracy": 0.65, "difference_pp": -4.0, "paired_cluster_bootstrap95_pp": [ -10.0, 1.0 ], "local_only_correct": 2, "jev_only_correct": 6, "both_correct": 59, "both_wrong": 33, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/support_topic": { "n": 100, "local_correct": 42, "jev_correct": 39, "local_accuracy": 0.42, "jev_accuracy": 0.39, "difference_pp": 3.0, "paired_cluster_bootstrap95_pp": [ -3.0, 10.0 ], "local_only_correct": 7, "jev_only_correct": 4, "both_correct": 35, "both_wrong": 54, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/ticket_route": { "n": 100, "local_correct": 13, "jev_correct": 9, "local_accuracy": 0.13, "jev_accuracy": 0.09, "difference_pp": 4.0, "paired_cluster_bootstrap95_pp": [ -2.0, 10.0 ], "local_only_correct": 7, "jev_only_correct": 3, "both_correct": 6, "both_wrong": 84, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "fast_decisions_dev/travel_request": { "n": 100, "local_correct": 56, "jev_correct": 59, "local_accuracy": 0.56, "jev_accuracy": 0.59, "difference_pp": -3.0, "paired_cluster_bootstrap95_pp": [ -8.0, 1.0 ], "local_only_correct": 1, "jev_only_correct": 4, "both_correct": 55, "both_wrong": 40, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/agnews": { "n": 100, "local_correct": 89, "jev_correct": 91, "local_accuracy": 0.89, "jev_accuracy": 0.91, "difference_pp": -2.0, "paired_cluster_bootstrap95_pp": [ -5.0, 0.0 ], "local_only_correct": 0, "jev_only_correct": 2, "both_correct": 89, "both_wrong": 9, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/banking77": { "n": 100, "local_correct": 0, "jev_correct": 88, "local_accuracy": 0.0, "jev_accuracy": 0.88, "difference_pp": -88.0, "paired_cluster_bootstrap95_pp": [ -94.0, -81.0 ], "local_only_correct": 0, "jev_only_correct": 88, "both_correct": 0, "both_wrong": 12, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "jev_native/emotiondair": { "n": 100, "local_correct": 50, "jev_correct": 48, "local_accuracy": 0.5, "jev_accuracy": 0.48, "difference_pp": 2.0, "paired_cluster_bootstrap95_pp": [ -5.0, 9.0 ], "local_only_correct": 7, "jev_only_correct": 5, "both_correct": 43, "both_wrong": 45, "independent_groups": 100, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_en": { "n": 2948, "local_correct": 2454, "jev_correct": 2550, "local_accuracy": 0.8324287652645862, "jev_accuracy": 0.8649932157394844, "difference_pp": -3.2564450474898234, "paired_cluster_bootstrap95_pp": [ -4.341926729986431, -2.2048846675712346 ], "local_only_correct": 81, "jev_only_correct": 177, "both_correct": 2373, "both_wrong": 317, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "massive_nb": { "n": 2948, "local_correct": 2299, "jev_correct": 2412, "local_accuracy": 0.7798507462686567, "jev_accuracy": 0.8181818181818182, "difference_pp": -3.8331071913161465, "paired_cluster_bootstrap95_pp": [ -5.020352781546811, -2.611940298507463 ], "local_only_correct": 119, "jev_only_correct": 232, "both_correct": 2180, "both_wrong": 417, "independent_groups": 2948, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false }, "norec_no": { "n": 1173, "local_correct": 863, "jev_correct": 843, "local_accuracy": 0.7357203751065644, "jev_accuracy": 0.7186700767263428, "difference_pp": 1.7050298380221656, "paired_cluster_bootstrap95_pp": [ -0.6270487017524157, 3.984662187856508 ], "local_only_correct": 103, "jev_only_correct": 83, "both_correct": 760, "both_wrong": 227, "independent_groups": 47, "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.", "degenerate_bootstrap": false } } }, "keyword_baseline": { "fresh_business/routing/en": { "n": 12, "correct": 4, "accuracy": 0.3333333333333333, "wilson95": [ 0.13812009109121315, 0.6093779111272004 ], "note": "Existing fixed keyword rules, no fitting. Label-only baseline: no invented confidence or calibration." }, "fresh_business/routing/nb": { "n": 12, "correct": 6, "accuracy": 0.5, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "note": "Existing fixed keyword rules, no fitting. Label-only baseline: no invented confidence or calibration." }, "fresh_business/documents/en": { "n": 12, "correct": 2, "accuracy": 0.16666666666666666, "wilson95": [ 0.04696514218385378, 0.4480308622529734 ], "note": "Existing fixed keyword rules, no fitting. Label-only baseline: no invented confidence or calibration." }, "fresh_business/documents/nb": { "n": 12, "correct": 3, "accuracy": 0.25, "wilson95": [ 0.08894166839405471, 0.5323053349335657 ], "note": "Existing fixed keyword rules, no fitting. Label-only baseline: no invented confidence or calibration." }, "industry/documents/en": { "n": 28, "correct": 11, "accuracy": 0.39285714285714285, "wilson95": [ 0.2356568654157959, 0.5759095570340281 ], "note": "Existing fixed keyword rules, no fitting. Label-only baseline: no invented confidence or calibration." }, "industry/documents/nb": { "n": 28, "correct": 15, "accuracy": 0.5357142857142857, "wilson95": [ 0.358126892344679, 0.7046843001720462 ], "note": "Existing fixed keyword rules, no fitting. Label-only baseline: no invented confidence or calibration." }, "industry/routing/en": { "n": 40, "correct": 18, "accuracy": 0.45, "wilson95": [ 0.3070530781090492, 0.6017090820106794 ], "note": "Existing fixed keyword rules, no fitting. Label-only baseline: no invented confidence or calibration." }, "industry/routing/nb": { "n": 40, "correct": 21, "accuracy": 0.525, "wilson95": [ 0.3749736210669248, 0.670645298873211 ], "note": "Existing fixed keyword rules, no fitting. Label-only baseline: no invented confidence or calibration." } }, "diagnostics": { "updated": "2026-10-01T08:44:07.727706+00:00", "expected": 312, "completed": 312, "by_variant": { "repeat1": { "n": 112, "valid": 112, "same_decision": 111, "changed_decisions": [ "jevbench_public/hard/hard-opus-c-temporal_numeric-12" ], "max_probability_change": 0.09000000000000002 }, "repeat2": { "n": 112, "valid": 112, "same_decision": 111, "changed_decisions": [ "jevbench_public/hard/hard-opus-c-temporal_numeric-12" ], "max_probability_change": 0.14 }, "reversed_choices": { "n": 88, "valid": 88, "same_decision": 87, "changed_decisions": [ "jevbench_public/hard/hard-opus-a-probability-07" ], "max_probability_change": 0.26 } } }, "priority_comparison_complete": true }